{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook explores an application of the LightGBM package by Microsoft with the a dataset on credit cards. Dataset can be found at: https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_excel('ccdata.xls', header = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>LIMIT_BAL</th>\n",
       "      <th>SEX</th>\n",
       "      <th>EDUCATION</th>\n",
       "      <th>MARRIAGE</th>\n",
       "      <th>AGE</th>\n",
       "      <th>PAY_0</th>\n",
       "      <th>PAY_2</th>\n",
       "      <th>PAY_3</th>\n",
       "      <th>PAY_4</th>\n",
       "      <th>...</th>\n",
       "      <th>BILL_AMT4</th>\n",
       "      <th>BILL_AMT5</th>\n",
       "      <th>BILL_AMT6</th>\n",
       "      <th>PAY_AMT1</th>\n",
       "      <th>PAY_AMT2</th>\n",
       "      <th>PAY_AMT3</th>\n",
       "      <th>PAY_AMT4</th>\n",
       "      <th>PAY_AMT5</th>\n",
       "      <th>PAY_AMT6</th>\n",
       "      <th>default payment next month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>20000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>689</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>120000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>-1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3272</td>\n",
       "      <td>3455</td>\n",
       "      <td>3261</td>\n",
       "      <td>0</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>90000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>14331</td>\n",
       "      <td>14948</td>\n",
       "      <td>15549</td>\n",
       "      <td>1518</td>\n",
       "      <td>1500</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>50000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>28314</td>\n",
       "      <td>28959</td>\n",
       "      <td>29547</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019</td>\n",
       "      <td>1200</td>\n",
       "      <td>1100</td>\n",
       "      <td>1069</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>50000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>20940</td>\n",
       "      <td>19146</td>\n",
       "      <td>19131</td>\n",
       "      <td>2000</td>\n",
       "      <td>36681</td>\n",
       "      <td>10000</td>\n",
       "      <td>9000</td>\n",
       "      <td>689</td>\n",
       "      <td>679</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  LIMIT_BAL  SEX  EDUCATION  MARRIAGE  AGE  PAY_0  PAY_2  PAY_3  PAY_4  \\\n",
       "0   1      20000    2          2         1   24      2      2     -1     -1   \n",
       "1   2     120000    2          2         2   26     -1      2      0      0   \n",
       "2   3      90000    2          2         2   34      0      0      0      0   \n",
       "3   4      50000    2          2         1   37      0      0      0      0   \n",
       "4   5      50000    1          2         1   57     -1      0     -1      0   \n",
       "\n",
       "              ...              BILL_AMT4  BILL_AMT5  BILL_AMT6  PAY_AMT1  \\\n",
       "0             ...                      0          0          0         0   \n",
       "1             ...                   3272       3455       3261         0   \n",
       "2             ...                  14331      14948      15549      1518   \n",
       "3             ...                  28314      28959      29547      2000   \n",
       "4             ...                  20940      19146      19131      2000   \n",
       "\n",
       "   PAY_AMT2  PAY_AMT3  PAY_AMT4  PAY_AMT5  PAY_AMT6  \\\n",
       "0       689         0         0         0         0   \n",
       "1      1000      1000      1000         0      2000   \n",
       "2      1500      1000      1000      1000      5000   \n",
       "3      2019      1200      1100      1069      1000   \n",
       "4     36681     10000      9000       689       679   \n",
       "\n",
       "   default payment next month  \n",
       "0                           1  \n",
       "1                           1  \n",
       "2                           0  \n",
       "3                           0  \n",
       "4                           0  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This research employed a binary variable, default payment (Yes = 1, No = 0), as the response variable. This study reviewed the literature and used the following 23 variables as explanatory variables:\n",
    "- X1: Amount of the given credit (NT dollar): it includes both the individual consumer credit and his/her family (supplementary) credit.\n",
    "- X2: Gender (1 = male; 2 = female).\n",
    "- X3: Education (1 = graduate school; 2 = university; 3 = high school; 4 = others).\n",
    "- X4: Marital status (1 = married; 2 = single; 3 = others).\n",
    "- X5: Age (year).\n",
    "- X6 - X11: History of past payment. We tracked the past monthly payment records (from April to September, 2005) as follows: X6 = the repayment status in September, 2005; X7 = the repayment status in August, 2005; . . .;X11 = the repayment status in April, 2005. The measurement scale for the repayment status is: -1 = pay duly; 1 = payment delay for one month; 2 = payment delay for two months; . . .; 8 = payment delay for eight months; 9 = payment delay for nine months and above.\n",
    "- X12-X17: Amount of bill statement (NT dollar). X12 = amount of bill statement in September, 2005; X13 = amount of bill statement in August, 2005; . . .; X17 = amount of bill statement in April, 2005.\n",
    "- X18-X23: Amount of previous payment (NT dollar). X18 = amount paid in September, 2005; X19 = amount paid in August, 2005; . . .;X23 = amount paid in April, 2005.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.drop('ID', axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "default payment next month    0\n",
       "PAY_AMT6                      0\n",
       "SEX                           0\n",
       "EDUCATION                     0\n",
       "MARRIAGE                      0\n",
       "AGE                           0\n",
       "PAY_0                         0\n",
       "PAY_2                         0\n",
       "PAY_3                         0\n",
       "PAY_4                         0\n",
       "PAY_5                         0\n",
       "PAY_6                         0\n",
       "BILL_AMT1                     0\n",
       "BILL_AMT2                     0\n",
       "BILL_AMT3                     0\n",
       "BILL_AMT4                     0\n",
       "BILL_AMT5                     0\n",
       "BILL_AMT6                     0\n",
       "PAY_AMT1                      0\n",
       "PAY_AMT2                      0\n",
       "PAY_AMT3                      0\n",
       "PAY_AMT4                      0\n",
       "PAY_AMT5                      0\n",
       "LIMIT_BAL                     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.drop(['default payment next month'], axis=1)\n",
    "y = data['default payment next month']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's binary_logloss: 0.493444\tvalid_0's auc: 0.769006\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471499\tvalid_0's auc: 0.774955\n",
      "[3]\tvalid_0's binary_logloss: 0.457955\tvalid_0's auc: 0.77727\n",
      "[4]\tvalid_0's binary_logloss: 0.448761\tvalid_0's auc: 0.777876\n",
      "[5]\tvalid_0's binary_logloss: 0.442876\tvalid_0's auc: 0.777897\n",
      "[6]\tvalid_0's binary_logloss: 0.438849\tvalid_0's auc: 0.778296\n",
      "[7]\tvalid_0's binary_logloss: 0.436218\tvalid_0's auc: 0.778106\n",
      "[8]\tvalid_0's binary_logloss: 0.433987\tvalid_0's auc: 0.779527\n",
      "[9]\tvalid_0's binary_logloss: 0.432826\tvalid_0's auc: 0.779085\n",
      "[10]\tvalid_0's binary_logloss: 0.432235\tvalid_0's auc: 0.778517\n",
      "[11]\tvalid_0's binary_logloss: 0.431738\tvalid_0's auc: 0.778939\n",
      "[12]\tvalid_0's binary_logloss: 0.430937\tvalid_0's auc: 0.779664\n",
      "[13]\tvalid_0's binary_logloss: 0.430385\tvalid_0's auc: 0.780715\n",
      "[14]\tvalid_0's binary_logloss: 0.430531\tvalid_0's auc: 0.780489\n",
      "[15]\tvalid_0's binary_logloss: 0.430717\tvalid_0's auc: 0.779554\n",
      "[16]\tvalid_0's binary_logloss: 0.430617\tvalid_0's auc: 0.779612\n",
      "[17]\tvalid_0's binary_logloss: 0.430684\tvalid_0's auc: 0.779369\n",
      "[18]\tvalid_0's binary_logloss: 0.431521\tvalid_0's auc: 0.779128\n",
      "Early stopping, best iteration is:\n",
      "[13]\tvalid_0's binary_logloss: 0.430385\tvalid_0's auc: 0.780715\n",
      "[1]\tvalid_0's binary_logloss: 0.492766\tvalid_0's auc: 0.768786\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470969\tvalid_0's auc: 0.772636\n",
      "[3]\tvalid_0's binary_logloss: 0.458056\tvalid_0's auc: 0.772786\n",
      "[4]\tvalid_0's binary_logloss: 0.448865\tvalid_0's auc: 0.775925\n",
      "[5]\tvalid_0's binary_logloss: 0.442984\tvalid_0's auc: 0.776371\n",
      "[6]\tvalid_0's binary_logloss: 0.439141\tvalid_0's auc: 0.776696\n",
      "[7]\tvalid_0's binary_logloss: 0.436468\tvalid_0's auc: 0.775879\n",
      "[8]\tvalid_0's binary_logloss: 0.43406\tvalid_0's auc: 0.777948\n",
      "[9]\tvalid_0's binary_logloss: 0.432295\tvalid_0's auc: 0.779407\n",
      "[10]\tvalid_0's binary_logloss: 0.430528\tvalid_0's auc: 0.781713\n",
      "[11]\tvalid_0's binary_logloss: 0.430079\tvalid_0's auc: 0.782022\n",
      "[12]\tvalid_0's binary_logloss: 0.429817\tvalid_0's auc: 0.782196\n",
      "[13]\tvalid_0's binary_logloss: 0.429536\tvalid_0's auc: 0.782493\n",
      "[14]\tvalid_0's binary_logloss: 0.429355\tvalid_0's auc: 0.782534\n",
      "[15]\tvalid_0's binary_logloss: 0.429616\tvalid_0's auc: 0.781537\n",
      "[16]\tvalid_0's binary_logloss: 0.42899\tvalid_0's auc: 0.783022\n",
      "[17]\tvalid_0's binary_logloss: 0.429222\tvalid_0's auc: 0.782245\n",
      "[18]\tvalid_0's binary_logloss: 0.429008\tvalid_0's auc: 0.782757\n",
      "[19]\tvalid_0's binary_logloss: 0.428806\tvalid_0's auc: 0.783323\n",
      "[20]\tvalid_0's binary_logloss: 0.42933\tvalid_0's auc: 0.782672\n",
      "[1]\tvalid_0's binary_logloss: 0.493905\tvalid_0's auc: 0.762943\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.47168\tvalid_0's auc: 0.774764\n",
      "[3]\tvalid_0's binary_logloss: 0.458436\tvalid_0's auc: 0.776083\n",
      "[4]\tvalid_0's binary_logloss: 0.448676\tvalid_0's auc: 0.779111\n",
      "[5]\tvalid_0's binary_logloss: 0.442676\tvalid_0's auc: 0.780125\n",
      "[6]\tvalid_0's binary_logloss: 0.438519\tvalid_0's auc: 0.780228\n",
      "[7]\tvalid_0's binary_logloss: 0.435181\tvalid_0's auc: 0.781089\n",
      "[8]\tvalid_0's binary_logloss: 0.432895\tvalid_0's auc: 0.78205\n",
      "[9]\tvalid_0's binary_logloss: 0.430973\tvalid_0's auc: 0.783574\n",
      "[10]\tvalid_0's binary_logloss: 0.429614\tvalid_0's auc: 0.784658\n",
      "[11]\tvalid_0's binary_logloss: 0.429158\tvalid_0's auc: 0.784562\n",
      "[12]\tvalid_0's binary_logloss: 0.429193\tvalid_0's auc: 0.784534\n",
      "[13]\tvalid_0's binary_logloss: 0.429192\tvalid_0's auc: 0.783636\n",
      "[14]\tvalid_0's binary_logloss: 0.429443\tvalid_0's auc: 0.782974\n",
      "[15]\tvalid_0's binary_logloss: 0.42953\tvalid_0's auc: 0.782676\n",
      "Early stopping, best iteration is:\n",
      "[10]\tvalid_0's binary_logloss: 0.429614\tvalid_0's auc: 0.784658\n",
      "[1]\tvalid_0's binary_logloss: 0.49335\tvalid_0's auc: 0.766065\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470984\tvalid_0's auc: 0.774251\n",
      "[3]\tvalid_0's binary_logloss: 0.457707\tvalid_0's auc: 0.775262\n",
      "[4]\tvalid_0's binary_logloss: 0.448575\tvalid_0's auc: 0.776878\n",
      "[5]\tvalid_0's binary_logloss: 0.442767\tvalid_0's auc: 0.776255\n",
      "[6]\tvalid_0's binary_logloss: 0.438366\tvalid_0's auc: 0.777217\n",
      "[7]\tvalid_0's binary_logloss: 0.435734\tvalid_0's auc: 0.777467\n",
      "[8]\tvalid_0's binary_logloss: 0.434446\tvalid_0's auc: 0.776672\n",
      "[9]\tvalid_0's binary_logloss: 0.433444\tvalid_0's auc: 0.776498\n",
      "[10]\tvalid_0's binary_logloss: 0.432953\tvalid_0's auc: 0.775806\n",
      "[11]\tvalid_0's binary_logloss: 0.432202\tvalid_0's auc: 0.776704\n",
      "[12]\tvalid_0's binary_logloss: 0.432178\tvalid_0's auc: 0.776486\n",
      "Early stopping, best iteration is:\n",
      "[7]\tvalid_0's binary_logloss: 0.435734\tvalid_0's auc: 0.777467\n",
      "[1]\tvalid_0's binary_logloss: 0.492897\tvalid_0's auc: 0.766887\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471064\tvalid_0's auc: 0.773704\n",
      "[3]\tvalid_0's binary_logloss: 0.457198\tvalid_0's auc: 0.774864\n",
      "[4]\tvalid_0's binary_logloss: 0.447779\tvalid_0's auc: 0.776975\n",
      "[5]\tvalid_0's binary_logloss: 0.441954\tvalid_0's auc: 0.778679\n",
      "[6]\tvalid_0's binary_logloss: 0.437462\tvalid_0's auc: 0.779706\n",
      "[7]\tvalid_0's binary_logloss: 0.434779\tvalid_0's auc: 0.780117\n",
      "[8]\tvalid_0's binary_logloss: 0.433528\tvalid_0's auc: 0.780178\n",
      "[9]\tvalid_0's binary_logloss: 0.432613\tvalid_0's auc: 0.780141\n",
      "[10]\tvalid_0's binary_logloss: 0.43157\tvalid_0's auc: 0.781131\n",
      "[11]\tvalid_0's binary_logloss: 0.431007\tvalid_0's auc: 0.78111\n",
      "[12]\tvalid_0's binary_logloss: 0.430777\tvalid_0's auc: 0.780598\n",
      "[13]\tvalid_0's binary_logloss: 0.430623\tvalid_0's auc: 0.780786\n",
      "[14]\tvalid_0's binary_logloss: 0.430218\tvalid_0's auc: 0.781265\n",
      "[15]\tvalid_0's binary_logloss: 0.430104\tvalid_0's auc: 0.781171\n",
      "[16]\tvalid_0's binary_logloss: 0.430421\tvalid_0's auc: 0.781122\n",
      "[17]\tvalid_0's binary_logloss: 0.430072\tvalid_0's auc: 0.781264\n",
      "[18]\tvalid_0's binary_logloss: 0.430331\tvalid_0's auc: 0.780148\n",
      "[19]\tvalid_0's binary_logloss: 0.430365\tvalid_0's auc: 0.780587\n",
      "Early stopping, best iteration is:\n",
      "[14]\tvalid_0's binary_logloss: 0.430218\tvalid_0's auc: 0.781265\n",
      "[1]\tvalid_0's binary_logloss: 0.493852\tvalid_0's auc: 0.762627\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471908\tvalid_0's auc: 0.772765\n",
      "[3]\tvalid_0's binary_logloss: 0.457457\tvalid_0's auc: 0.775748\n",
      "[4]\tvalid_0's binary_logloss: 0.448214\tvalid_0's auc: 0.778358\n",
      "[5]\tvalid_0's binary_logloss: 0.442011\tvalid_0's auc: 0.778796\n",
      "[6]\tvalid_0's binary_logloss: 0.437874\tvalid_0's auc: 0.779648\n",
      "[7]\tvalid_0's binary_logloss: 0.434842\tvalid_0's auc: 0.781034\n",
      "[8]\tvalid_0's binary_logloss: 0.433134\tvalid_0's auc: 0.781353\n",
      "[9]\tvalid_0's binary_logloss: 0.431642\tvalid_0's auc: 0.781921\n",
      "[10]\tvalid_0's binary_logloss: 0.431359\tvalid_0's auc: 0.780833\n",
      "[11]\tvalid_0's binary_logloss: 0.430703\tvalid_0's auc: 0.780926\n",
      "[12]\tvalid_0's binary_logloss: 0.430246\tvalid_0's auc: 0.781071\n",
      "[13]\tvalid_0's binary_logloss: 0.4302\tvalid_0's auc: 0.780928\n",
      "[14]\tvalid_0's binary_logloss: 0.430064\tvalid_0's auc: 0.780671\n",
      "Early stopping, best iteration is:\n",
      "[9]\tvalid_0's binary_logloss: 0.431642\tvalid_0's auc: 0.781921\n",
      "[1]\tvalid_0's binary_logloss: 0.493065\tvalid_0's auc: 0.765444\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470627\tvalid_0's auc: 0.773937\n",
      "[3]\tvalid_0's binary_logloss: 0.45709\tvalid_0's auc: 0.775908\n",
      "[4]\tvalid_0's binary_logloss: 0.447431\tvalid_0's auc: 0.777401\n",
      "[5]\tvalid_0's binary_logloss: 0.441719\tvalid_0's auc: 0.776322\n",
      "[6]\tvalid_0's binary_logloss: 0.438067\tvalid_0's auc: 0.777719\n",
      "[7]\tvalid_0's binary_logloss: 0.435234\tvalid_0's auc: 0.778062\n",
      "[8]\tvalid_0's binary_logloss: 0.433423\tvalid_0's auc: 0.77794\n",
      "[9]\tvalid_0's binary_logloss: 0.433098\tvalid_0's auc: 0.776908\n",
      "[10]\tvalid_0's binary_logloss: 0.432888\tvalid_0's auc: 0.776321\n",
      "[11]\tvalid_0's binary_logloss: 0.432195\tvalid_0's auc: 0.776818\n",
      "[12]\tvalid_0's binary_logloss: 0.432526\tvalid_0's auc: 0.776269\n",
      "Early stopping, best iteration is:\n",
      "[7]\tvalid_0's binary_logloss: 0.435234\tvalid_0's auc: 0.778062\n",
      "[1]\tvalid_0's binary_logloss: 0.492663\tvalid_0's auc: 0.767706\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470703\tvalid_0's auc: 0.770887\n",
      "[3]\tvalid_0's binary_logloss: 0.45666\tvalid_0's auc: 0.776062\n",
      "[4]\tvalid_0's binary_logloss: 0.447127\tvalid_0's auc: 0.779357\n",
      "[5]\tvalid_0's binary_logloss: 0.441263\tvalid_0's auc: 0.779819\n",
      "[6]\tvalid_0's binary_logloss: 0.436977\tvalid_0's auc: 0.780716\n",
      "[7]\tvalid_0's binary_logloss: 0.43418\tvalid_0's auc: 0.781167\n",
      "[8]\tvalid_0's binary_logloss: 0.432425\tvalid_0's auc: 0.781344\n",
      "[9]\tvalid_0's binary_logloss: 0.432164\tvalid_0's auc: 0.77986\n",
      "[10]\tvalid_0's binary_logloss: 0.431548\tvalid_0's auc: 0.7801\n",
      "[11]\tvalid_0's binary_logloss: 0.43138\tvalid_0's auc: 0.780085\n",
      "[12]\tvalid_0's binary_logloss: 0.429832\tvalid_0's auc: 0.781662\n",
      "[13]\tvalid_0's binary_logloss: 0.430143\tvalid_0's auc: 0.780935\n",
      "[14]\tvalid_0's binary_logloss: 0.429776\tvalid_0's auc: 0.781686\n",
      "[15]\tvalid_0's binary_logloss: 0.430472\tvalid_0's auc: 0.780877\n",
      "[16]\tvalid_0's binary_logloss: 0.430723\tvalid_0's auc: 0.780833\n",
      "[17]\tvalid_0's binary_logloss: 0.431451\tvalid_0's auc: 0.779975\n",
      "[18]\tvalid_0's binary_logloss: 0.432035\tvalid_0's auc: 0.778887\n",
      "[19]\tvalid_0's binary_logloss: 0.432584\tvalid_0's auc: 0.777891\n",
      "Early stopping, best iteration is:\n",
      "[14]\tvalid_0's binary_logloss: 0.429776\tvalid_0's auc: 0.781686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's binary_logloss: 0.493471\tvalid_0's auc: 0.761564\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471973\tvalid_0's auc: 0.772261\n",
      "[3]\tvalid_0's binary_logloss: 0.458114\tvalid_0's auc: 0.776067\n",
      "[4]\tvalid_0's binary_logloss: 0.44892\tvalid_0's auc: 0.777744\n",
      "[5]\tvalid_0's binary_logloss: 0.443273\tvalid_0's auc: 0.777383\n",
      "[6]\tvalid_0's binary_logloss: 0.440384\tvalid_0's auc: 0.777131\n",
      "[7]\tvalid_0's binary_logloss: 0.438101\tvalid_0's auc: 0.777756\n",
      "[8]\tvalid_0's binary_logloss: 0.435683\tvalid_0's auc: 0.779115\n",
      "[9]\tvalid_0's binary_logloss: 0.434615\tvalid_0's auc: 0.779148\n",
      "[10]\tvalid_0's binary_logloss: 0.434088\tvalid_0's auc: 0.778906\n",
      "[11]\tvalid_0's binary_logloss: 0.434039\tvalid_0's auc: 0.77889\n",
      "[12]\tvalid_0's binary_logloss: 0.434115\tvalid_0's auc: 0.778763\n",
      "[13]\tvalid_0's binary_logloss: 0.434103\tvalid_0's auc: 0.778776\n",
      "[14]\tvalid_0's binary_logloss: 0.434238\tvalid_0's auc: 0.778433\n",
      "Early stopping, best iteration is:\n",
      "[9]\tvalid_0's binary_logloss: 0.434615\tvalid_0's auc: 0.779148\n",
      "[1]\tvalid_0's binary_logloss: 0.492915\tvalid_0's auc: 0.763406\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470651\tvalid_0's auc: 0.77441\n",
      "[3]\tvalid_0's binary_logloss: 0.456593\tvalid_0's auc: 0.777957\n",
      "[4]\tvalid_0's binary_logloss: 0.448268\tvalid_0's auc: 0.776308\n",
      "[5]\tvalid_0's binary_logloss: 0.442045\tvalid_0's auc: 0.777587\n",
      "[6]\tvalid_0's binary_logloss: 0.43784\tvalid_0's auc: 0.778157\n",
      "[7]\tvalid_0's binary_logloss: 0.435109\tvalid_0's auc: 0.77969\n",
      "[8]\tvalid_0's binary_logloss: 0.434039\tvalid_0's auc: 0.778134\n",
      "[9]\tvalid_0's binary_logloss: 0.433347\tvalid_0's auc: 0.777759\n",
      "[10]\tvalid_0's binary_logloss: 0.432726\tvalid_0's auc: 0.777547\n",
      "[11]\tvalid_0's binary_logloss: 0.431991\tvalid_0's auc: 0.778308\n",
      "[12]\tvalid_0's binary_logloss: 0.432261\tvalid_0's auc: 0.777729\n",
      "Early stopping, best iteration is:\n",
      "[7]\tvalid_0's binary_logloss: 0.435109\tvalid_0's auc: 0.77969\n",
      "[1]\tvalid_0's binary_logloss: 0.493008\tvalid_0's auc: 0.765697\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470875\tvalid_0's auc: 0.772962\n",
      "[3]\tvalid_0's binary_logloss: 0.456731\tvalid_0's auc: 0.774598\n",
      "[4]\tvalid_0's binary_logloss: 0.44783\tvalid_0's auc: 0.775002\n",
      "[5]\tvalid_0's binary_logloss: 0.441115\tvalid_0's auc: 0.778411\n",
      "[6]\tvalid_0's binary_logloss: 0.437104\tvalid_0's auc: 0.778325\n",
      "[7]\tvalid_0's binary_logloss: 0.434681\tvalid_0's auc: 0.779302\n",
      "[8]\tvalid_0's binary_logloss: 0.433313\tvalid_0's auc: 0.779531\n",
      "[9]\tvalid_0's binary_logloss: 0.433019\tvalid_0's auc: 0.778046\n",
      "[10]\tvalid_0's binary_logloss: 0.432359\tvalid_0's auc: 0.777901\n",
      "[11]\tvalid_0's binary_logloss: 0.432413\tvalid_0's auc: 0.776446\n",
      "[12]\tvalid_0's binary_logloss: 0.432407\tvalid_0's auc: 0.776237\n",
      "[13]\tvalid_0's binary_logloss: 0.432312\tvalid_0's auc: 0.77666\n",
      "Early stopping, best iteration is:\n",
      "[8]\tvalid_0's binary_logloss: 0.433313\tvalid_0's auc: 0.779531\n",
      "[1]\tvalid_0's binary_logloss: 0.493763\tvalid_0's auc: 0.756607\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471979\tvalid_0's auc: 0.7709\n",
      "[3]\tvalid_0's binary_logloss: 0.457901\tvalid_0's auc: 0.775641\n",
      "[4]\tvalid_0's binary_logloss: 0.448801\tvalid_0's auc: 0.778662\n",
      "[5]\tvalid_0's binary_logloss: 0.442736\tvalid_0's auc: 0.778983\n",
      "[6]\tvalid_0's binary_logloss: 0.438823\tvalid_0's auc: 0.780309\n",
      "[7]\tvalid_0's binary_logloss: 0.43653\tvalid_0's auc: 0.780616\n",
      "[8]\tvalid_0's binary_logloss: 0.434954\tvalid_0's auc: 0.780663\n",
      "[9]\tvalid_0's binary_logloss: 0.433835\tvalid_0's auc: 0.78043\n",
      "[10]\tvalid_0's binary_logloss: 0.433373\tvalid_0's auc: 0.780383\n",
      "[11]\tvalid_0's binary_logloss: 0.433292\tvalid_0's auc: 0.780129\n",
      "[12]\tvalid_0's binary_logloss: 0.432992\tvalid_0's auc: 0.779996\n",
      "[13]\tvalid_0's binary_logloss: 0.433175\tvalid_0's auc: 0.780038\n",
      "Early stopping, best iteration is:\n",
      "[8]\tvalid_0's binary_logloss: 0.434954\tvalid_0's auc: 0.780663\n",
      "[1]\tvalid_0's binary_logloss: 0.493444\tvalid_0's auc: 0.769006\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471499\tvalid_0's auc: 0.774955\n",
      "[3]\tvalid_0's binary_logloss: 0.457955\tvalid_0's auc: 0.77727\n",
      "[4]\tvalid_0's binary_logloss: 0.448761\tvalid_0's auc: 0.777876\n",
      "[5]\tvalid_0's binary_logloss: 0.442876\tvalid_0's auc: 0.777897\n",
      "[6]\tvalid_0's binary_logloss: 0.438849\tvalid_0's auc: 0.778296\n",
      "[7]\tvalid_0's binary_logloss: 0.436218\tvalid_0's auc: 0.778106\n",
      "[8]\tvalid_0's binary_logloss: 0.433987\tvalid_0's auc: 0.779527\n",
      "[9]\tvalid_0's binary_logloss: 0.432826\tvalid_0's auc: 0.779085\n",
      "[10]\tvalid_0's binary_logloss: 0.432235\tvalid_0's auc: 0.778517\n",
      "[11]\tvalid_0's binary_logloss: 0.431738\tvalid_0's auc: 0.778939\n",
      "[12]\tvalid_0's binary_logloss: 0.430937\tvalid_0's auc: 0.779664\n",
      "[13]\tvalid_0's binary_logloss: 0.430385\tvalid_0's auc: 0.780715\n",
      "[14]\tvalid_0's binary_logloss: 0.430531\tvalid_0's auc: 0.780489\n",
      "[15]\tvalid_0's binary_logloss: 0.430717\tvalid_0's auc: 0.779554\n",
      "[16]\tvalid_0's binary_logloss: 0.430617\tvalid_0's auc: 0.779612\n",
      "[17]\tvalid_0's binary_logloss: 0.430684\tvalid_0's auc: 0.779369\n",
      "[18]\tvalid_0's binary_logloss: 0.431521\tvalid_0's auc: 0.779128\n",
      "Early stopping, best iteration is:\n",
      "[13]\tvalid_0's binary_logloss: 0.430385\tvalid_0's auc: 0.780715\n",
      "[1]\tvalid_0's binary_logloss: 0.492766\tvalid_0's auc: 0.768786\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470969\tvalid_0's auc: 0.772636\n",
      "[3]\tvalid_0's binary_logloss: 0.458056\tvalid_0's auc: 0.772786\n",
      "[4]\tvalid_0's binary_logloss: 0.448865\tvalid_0's auc: 0.775925\n",
      "[5]\tvalid_0's binary_logloss: 0.442984\tvalid_0's auc: 0.776371\n",
      "[6]\tvalid_0's binary_logloss: 0.439141\tvalid_0's auc: 0.776696\n",
      "[7]\tvalid_0's binary_logloss: 0.436468\tvalid_0's auc: 0.775879\n",
      "[8]\tvalid_0's binary_logloss: 0.43406\tvalid_0's auc: 0.777948\n",
      "[9]\tvalid_0's binary_logloss: 0.432295\tvalid_0's auc: 0.779407\n",
      "[10]\tvalid_0's binary_logloss: 0.430528\tvalid_0's auc: 0.781713\n",
      "[11]\tvalid_0's binary_logloss: 0.430079\tvalid_0's auc: 0.782022\n",
      "[12]\tvalid_0's binary_logloss: 0.429817\tvalid_0's auc: 0.782196\n",
      "[13]\tvalid_0's binary_logloss: 0.429536\tvalid_0's auc: 0.782493\n",
      "[14]\tvalid_0's binary_logloss: 0.429355\tvalid_0's auc: 0.782534\n",
      "[15]\tvalid_0's binary_logloss: 0.429616\tvalid_0's auc: 0.781537\n",
      "[16]\tvalid_0's binary_logloss: 0.42899\tvalid_0's auc: 0.783022\n",
      "[17]\tvalid_0's binary_logloss: 0.429222\tvalid_0's auc: 0.782245\n",
      "[18]\tvalid_0's binary_logloss: 0.429008\tvalid_0's auc: 0.782757\n",
      "[19]\tvalid_0's binary_logloss: 0.428806\tvalid_0's auc: 0.783323\n",
      "[20]\tvalid_0's binary_logloss: 0.42933\tvalid_0's auc: 0.782672\n",
      "[1]\tvalid_0's binary_logloss: 0.493905\tvalid_0's auc: 0.762943\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.47168\tvalid_0's auc: 0.774764\n",
      "[3]\tvalid_0's binary_logloss: 0.458436\tvalid_0's auc: 0.776083\n",
      "[4]\tvalid_0's binary_logloss: 0.448676\tvalid_0's auc: 0.779111\n",
      "[5]\tvalid_0's binary_logloss: 0.442676\tvalid_0's auc: 0.780125\n",
      "[6]\tvalid_0's binary_logloss: 0.438519\tvalid_0's auc: 0.780228\n",
      "[7]\tvalid_0's binary_logloss: 0.435181\tvalid_0's auc: 0.781089\n",
      "[8]\tvalid_0's binary_logloss: 0.432895\tvalid_0's auc: 0.78205\n",
      "[9]\tvalid_0's binary_logloss: 0.430973\tvalid_0's auc: 0.783574\n",
      "[10]\tvalid_0's binary_logloss: 0.429614\tvalid_0's auc: 0.784658\n",
      "[11]\tvalid_0's binary_logloss: 0.429158\tvalid_0's auc: 0.784562\n",
      "[12]\tvalid_0's binary_logloss: 0.429193\tvalid_0's auc: 0.784534\n",
      "[13]\tvalid_0's binary_logloss: 0.429192\tvalid_0's auc: 0.783636\n",
      "[14]\tvalid_0's binary_logloss: 0.429443\tvalid_0's auc: 0.782974\n",
      "[15]\tvalid_0's binary_logloss: 0.42953\tvalid_0's auc: 0.782676\n",
      "Early stopping, best iteration is:\n",
      "[10]\tvalid_0's binary_logloss: 0.429614\tvalid_0's auc: 0.784658\n",
      "[1]\tvalid_0's binary_logloss: 0.49335\tvalid_0's auc: 0.766065\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470984\tvalid_0's auc: 0.774251\n",
      "[3]\tvalid_0's binary_logloss: 0.457707\tvalid_0's auc: 0.775262\n",
      "[4]\tvalid_0's binary_logloss: 0.448575\tvalid_0's auc: 0.776878\n",
      "[5]\tvalid_0's binary_logloss: 0.442767\tvalid_0's auc: 0.776255\n",
      "[6]\tvalid_0's binary_logloss: 0.438366\tvalid_0's auc: 0.777217\n",
      "[7]\tvalid_0's binary_logloss: 0.435734\tvalid_0's auc: 0.777467\n",
      "[8]\tvalid_0's binary_logloss: 0.434446\tvalid_0's auc: 0.776672\n",
      "[9]\tvalid_0's binary_logloss: 0.433444\tvalid_0's auc: 0.776498\n",
      "[10]\tvalid_0's binary_logloss: 0.432953\tvalid_0's auc: 0.775806\n",
      "[11]\tvalid_0's binary_logloss: 0.432202\tvalid_0's auc: 0.776704\n",
      "[12]\tvalid_0's binary_logloss: 0.432178\tvalid_0's auc: 0.776486\n",
      "Early stopping, best iteration is:\n",
      "[7]\tvalid_0's binary_logloss: 0.435734\tvalid_0's auc: 0.777467\n",
      "[1]\tvalid_0's binary_logloss: 0.492897\tvalid_0's auc: 0.766887\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471064\tvalid_0's auc: 0.773704\n",
      "[3]\tvalid_0's binary_logloss: 0.457198\tvalid_0's auc: 0.774864\n",
      "[4]\tvalid_0's binary_logloss: 0.447779\tvalid_0's auc: 0.776975\n",
      "[5]\tvalid_0's binary_logloss: 0.441954\tvalid_0's auc: 0.778679\n",
      "[6]\tvalid_0's binary_logloss: 0.437462\tvalid_0's auc: 0.779706\n",
      "[7]\tvalid_0's binary_logloss: 0.434779\tvalid_0's auc: 0.780117\n",
      "[8]\tvalid_0's binary_logloss: 0.433528\tvalid_0's auc: 0.780178\n",
      "[9]\tvalid_0's binary_logloss: 0.432613\tvalid_0's auc: 0.780141\n",
      "[10]\tvalid_0's binary_logloss: 0.43157\tvalid_0's auc: 0.781131\n",
      "[11]\tvalid_0's binary_logloss: 0.431007\tvalid_0's auc: 0.78111\n",
      "[12]\tvalid_0's binary_logloss: 0.430777\tvalid_0's auc: 0.780598\n",
      "[13]\tvalid_0's binary_logloss: 0.430623\tvalid_0's auc: 0.780786\n",
      "[14]\tvalid_0's binary_logloss: 0.430218\tvalid_0's auc: 0.781265\n",
      "[15]\tvalid_0's binary_logloss: 0.430104\tvalid_0's auc: 0.781171\n",
      "[16]\tvalid_0's binary_logloss: 0.430421\tvalid_0's auc: 0.781122\n",
      "[17]\tvalid_0's binary_logloss: 0.430072\tvalid_0's auc: 0.781264\n",
      "[18]\tvalid_0's binary_logloss: 0.430331\tvalid_0's auc: 0.780148\n",
      "[19]\tvalid_0's binary_logloss: 0.430365\tvalid_0's auc: 0.780587\n",
      "Early stopping, best iteration is:\n",
      "[14]\tvalid_0's binary_logloss: 0.430218\tvalid_0's auc: 0.781265\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's binary_logloss: 0.493852\tvalid_0's auc: 0.762627\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471908\tvalid_0's auc: 0.772765\n",
      "[3]\tvalid_0's binary_logloss: 0.457457\tvalid_0's auc: 0.775748\n",
      "[4]\tvalid_0's binary_logloss: 0.448214\tvalid_0's auc: 0.778358\n",
      "[5]\tvalid_0's binary_logloss: 0.442011\tvalid_0's auc: 0.778796\n",
      "[6]\tvalid_0's binary_logloss: 0.437874\tvalid_0's auc: 0.779648\n",
      "[7]\tvalid_0's binary_logloss: 0.434842\tvalid_0's auc: 0.781034\n",
      "[8]\tvalid_0's binary_logloss: 0.433134\tvalid_0's auc: 0.781353\n",
      "[9]\tvalid_0's binary_logloss: 0.431642\tvalid_0's auc: 0.781921\n",
      "[10]\tvalid_0's binary_logloss: 0.431359\tvalid_0's auc: 0.780833\n",
      "[11]\tvalid_0's binary_logloss: 0.430703\tvalid_0's auc: 0.780926\n",
      "[12]\tvalid_0's binary_logloss: 0.430246\tvalid_0's auc: 0.781071\n",
      "[13]\tvalid_0's binary_logloss: 0.4302\tvalid_0's auc: 0.780928\n",
      "[14]\tvalid_0's binary_logloss: 0.430064\tvalid_0's auc: 0.780671\n",
      "Early stopping, best iteration is:\n",
      "[9]\tvalid_0's binary_logloss: 0.431642\tvalid_0's auc: 0.781921\n",
      "[1]\tvalid_0's binary_logloss: 0.493065\tvalid_0's auc: 0.765444\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470627\tvalid_0's auc: 0.773937\n",
      "[3]\tvalid_0's binary_logloss: 0.45709\tvalid_0's auc: 0.775908\n",
      "[4]\tvalid_0's binary_logloss: 0.447431\tvalid_0's auc: 0.777401\n",
      "[5]\tvalid_0's binary_logloss: 0.441719\tvalid_0's auc: 0.776322\n",
      "[6]\tvalid_0's binary_logloss: 0.438067\tvalid_0's auc: 0.777719\n",
      "[7]\tvalid_0's binary_logloss: 0.435234\tvalid_0's auc: 0.778062\n",
      "[8]\tvalid_0's binary_logloss: 0.433423\tvalid_0's auc: 0.77794\n",
      "[9]\tvalid_0's binary_logloss: 0.433098\tvalid_0's auc: 0.776908\n",
      "[10]\tvalid_0's binary_logloss: 0.432888\tvalid_0's auc: 0.776321\n",
      "[11]\tvalid_0's binary_logloss: 0.432195\tvalid_0's auc: 0.776818\n",
      "[12]\tvalid_0's binary_logloss: 0.432526\tvalid_0's auc: 0.776269\n",
      "Early stopping, best iteration is:\n",
      "[7]\tvalid_0's binary_logloss: 0.435234\tvalid_0's auc: 0.778062\n",
      "[1]\tvalid_0's binary_logloss: 0.492663\tvalid_0's auc: 0.767706\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470703\tvalid_0's auc: 0.770887\n",
      "[3]\tvalid_0's binary_logloss: 0.45666\tvalid_0's auc: 0.776062\n",
      "[4]\tvalid_0's binary_logloss: 0.447127\tvalid_0's auc: 0.779357\n",
      "[5]\tvalid_0's binary_logloss: 0.441263\tvalid_0's auc: 0.779819\n",
      "[6]\tvalid_0's binary_logloss: 0.436977\tvalid_0's auc: 0.780716\n",
      "[7]\tvalid_0's binary_logloss: 0.43418\tvalid_0's auc: 0.781167\n",
      "[8]\tvalid_0's binary_logloss: 0.432425\tvalid_0's auc: 0.781344\n",
      "[9]\tvalid_0's binary_logloss: 0.432164\tvalid_0's auc: 0.77986\n",
      "[10]\tvalid_0's binary_logloss: 0.431548\tvalid_0's auc: 0.7801\n",
      "[11]\tvalid_0's binary_logloss: 0.43138\tvalid_0's auc: 0.780085\n",
      "[12]\tvalid_0's binary_logloss: 0.429832\tvalid_0's auc: 0.781662\n",
      "[13]\tvalid_0's binary_logloss: 0.430143\tvalid_0's auc: 0.780935\n",
      "[14]\tvalid_0's binary_logloss: 0.429776\tvalid_0's auc: 0.781686\n",
      "[15]\tvalid_0's binary_logloss: 0.430472\tvalid_0's auc: 0.780877\n",
      "[16]\tvalid_0's binary_logloss: 0.430723\tvalid_0's auc: 0.780833\n",
      "[17]\tvalid_0's binary_logloss: 0.431451\tvalid_0's auc: 0.779975\n",
      "[18]\tvalid_0's binary_logloss: 0.432035\tvalid_0's auc: 0.778887\n",
      "[19]\tvalid_0's binary_logloss: 0.432584\tvalid_0's auc: 0.777891\n",
      "Early stopping, best iteration is:\n",
      "[14]\tvalid_0's binary_logloss: 0.429776\tvalid_0's auc: 0.781686\n",
      "[1]\tvalid_0's binary_logloss: 0.493471\tvalid_0's auc: 0.761564\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471973\tvalid_0's auc: 0.772261\n",
      "[3]\tvalid_0's binary_logloss: 0.458114\tvalid_0's auc: 0.776067\n",
      "[4]\tvalid_0's binary_logloss: 0.44892\tvalid_0's auc: 0.777744\n",
      "[5]\tvalid_0's binary_logloss: 0.443273\tvalid_0's auc: 0.777383\n",
      "[6]\tvalid_0's binary_logloss: 0.440384\tvalid_0's auc: 0.777131\n",
      "[7]\tvalid_0's binary_logloss: 0.438101\tvalid_0's auc: 0.777756\n",
      "[8]\tvalid_0's binary_logloss: 0.435683\tvalid_0's auc: 0.779115\n",
      "[9]\tvalid_0's binary_logloss: 0.434615\tvalid_0's auc: 0.779148\n",
      "[10]\tvalid_0's binary_logloss: 0.434088\tvalid_0's auc: 0.778906\n",
      "[11]\tvalid_0's binary_logloss: 0.434039\tvalid_0's auc: 0.77889\n",
      "[12]\tvalid_0's binary_logloss: 0.434115\tvalid_0's auc: 0.778763\n",
      "[13]\tvalid_0's binary_logloss: 0.434103\tvalid_0's auc: 0.778776\n",
      "[14]\tvalid_0's binary_logloss: 0.434238\tvalid_0's auc: 0.778433\n",
      "Early stopping, best iteration is:\n",
      "[9]\tvalid_0's binary_logloss: 0.434615\tvalid_0's auc: 0.779148\n",
      "[1]\tvalid_0's binary_logloss: 0.492915\tvalid_0's auc: 0.763406\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470651\tvalid_0's auc: 0.77441\n",
      "[3]\tvalid_0's binary_logloss: 0.456593\tvalid_0's auc: 0.777957\n",
      "[4]\tvalid_0's binary_logloss: 0.448268\tvalid_0's auc: 0.776308\n",
      "[5]\tvalid_0's binary_logloss: 0.442045\tvalid_0's auc: 0.777587\n",
      "[6]\tvalid_0's binary_logloss: 0.43784\tvalid_0's auc: 0.778157\n",
      "[7]\tvalid_0's binary_logloss: 0.435109\tvalid_0's auc: 0.77969\n",
      "[8]\tvalid_0's binary_logloss: 0.434039\tvalid_0's auc: 0.778134\n",
      "[9]\tvalid_0's binary_logloss: 0.433347\tvalid_0's auc: 0.777759\n",
      "[10]\tvalid_0's binary_logloss: 0.432726\tvalid_0's auc: 0.777547\n",
      "[11]\tvalid_0's binary_logloss: 0.431991\tvalid_0's auc: 0.778308\n",
      "[12]\tvalid_0's binary_logloss: 0.432261\tvalid_0's auc: 0.777729\n",
      "Early stopping, best iteration is:\n",
      "[7]\tvalid_0's binary_logloss: 0.435109\tvalid_0's auc: 0.77969\n",
      "[1]\tvalid_0's binary_logloss: 0.493008\tvalid_0's auc: 0.765697\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.470875\tvalid_0's auc: 0.772962\n",
      "[3]\tvalid_0's binary_logloss: 0.456731\tvalid_0's auc: 0.774598\n",
      "[4]\tvalid_0's binary_logloss: 0.44783\tvalid_0's auc: 0.775002\n",
      "[5]\tvalid_0's binary_logloss: 0.441115\tvalid_0's auc: 0.778411\n",
      "[6]\tvalid_0's binary_logloss: 0.437104\tvalid_0's auc: 0.778325\n",
      "[7]\tvalid_0's binary_logloss: 0.434681\tvalid_0's auc: 0.779302\n",
      "[8]\tvalid_0's binary_logloss: 0.433313\tvalid_0's auc: 0.779531\n",
      "[9]\tvalid_0's binary_logloss: 0.433019\tvalid_0's auc: 0.778046\n",
      "[10]\tvalid_0's binary_logloss: 0.432359\tvalid_0's auc: 0.777901\n",
      "[11]\tvalid_0's binary_logloss: 0.432413\tvalid_0's auc: 0.776446\n",
      "[12]\tvalid_0's binary_logloss: 0.432407\tvalid_0's auc: 0.776237\n",
      "[13]\tvalid_0's binary_logloss: 0.432312\tvalid_0's auc: 0.77666\n",
      "Early stopping, best iteration is:\n",
      "[8]\tvalid_0's binary_logloss: 0.433313\tvalid_0's auc: 0.779531\n",
      "[1]\tvalid_0's binary_logloss: 0.493763\tvalid_0's auc: 0.756607\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471979\tvalid_0's auc: 0.7709\n",
      "[3]\tvalid_0's binary_logloss: 0.457901\tvalid_0's auc: 0.775641\n",
      "[4]\tvalid_0's binary_logloss: 0.448801\tvalid_0's auc: 0.778662\n",
      "[5]\tvalid_0's binary_logloss: 0.442736\tvalid_0's auc: 0.778983\n",
      "[6]\tvalid_0's binary_logloss: 0.438823\tvalid_0's auc: 0.780309\n",
      "[7]\tvalid_0's binary_logloss: 0.43653\tvalid_0's auc: 0.780616\n",
      "[8]\tvalid_0's binary_logloss: 0.434954\tvalid_0's auc: 0.780663\n",
      "[9]\tvalid_0's binary_logloss: 0.433835\tvalid_0's auc: 0.78043\n",
      "[10]\tvalid_0's binary_logloss: 0.433373\tvalid_0's auc: 0.780383\n",
      "[11]\tvalid_0's binary_logloss: 0.433292\tvalid_0's auc: 0.780129\n",
      "[12]\tvalid_0's binary_logloss: 0.432992\tvalid_0's auc: 0.779996\n",
      "[13]\tvalid_0's binary_logloss: 0.433175\tvalid_0's auc: 0.780038\n",
      "Early stopping, best iteration is:\n",
      "[8]\tvalid_0's binary_logloss: 0.434954\tvalid_0's auc: 0.780663\n",
      "[1]\tvalid_0's binary_logloss: 0.49342\tvalid_0's auc: 0.770333\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.471065\tvalid_0's auc: 0.776688\n",
      "[3]\tvalid_0's binary_logloss: 0.456884\tvalid_0's auc: 0.778446\n",
      "[4]\tvalid_0's binary_logloss: 0.447784\tvalid_0's auc: 0.780452\n",
      "[5]\tvalid_0's binary_logloss: 0.441506\tvalid_0's auc: 0.781266\n",
      "[6]\tvalid_0's binary_logloss: 0.437053\tvalid_0's auc: 0.781685\n",
      "[7]\tvalid_0's binary_logloss: 0.434077\tvalid_0's auc: 0.782476\n",
      "[8]\tvalid_0's binary_logloss: 0.432288\tvalid_0's auc: 0.782362\n",
      "[9]\tvalid_0's binary_logloss: 0.431294\tvalid_0's auc: 0.782275\n",
      "[10]\tvalid_0's binary_logloss: 0.430215\tvalid_0's auc: 0.782478\n",
      "[11]\tvalid_0's binary_logloss: 0.429897\tvalid_0's auc: 0.781949\n",
      "[12]\tvalid_0's binary_logloss: 0.429323\tvalid_0's auc: 0.782068\n",
      "[13]\tvalid_0's binary_logloss: 0.428967\tvalid_0's auc: 0.782298\n",
      "[14]\tvalid_0's binary_logloss: 0.428866\tvalid_0's auc: 0.782115\n",
      "[15]\tvalid_0's binary_logloss: 0.428614\tvalid_0's auc: 0.781974\n",
      "Early stopping, best iteration is:\n",
      "[10]\tvalid_0's binary_logloss: 0.430215\tvalid_0's auc: 0.782478\n",
      "Best parameters found by grid search are: {'metric': 'l1', 'num_leaves': 30}\n"
     ]
    }
   ],
   "source": [
    "estimator = lgb.LGBMRegressor(objective='binary',\n",
    "                        num_leaves=31, learning_rate = 0.2, n_estimators = 20)\n",
    "param_grid = {\n",
    "    'num_leaves': [x for x in range(30, 70, 10)],\n",
    "    'metric': ('l1', 'l2')\n",
    "}\n",
    "gridsearch = GridSearchCV(estimator, param_grid)\n",
    "\n",
    "gridsearch.fit(X_train, y_train,\n",
    "        eval_set=[(X_test, y_test)],\n",
    "        eval_metric=['auc', 'binary_logloss'],\n",
    "early_stopping_rounds=5)\n",
    "\n",
    "print('Best parameters found by grid search are:', gridsearch.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's binary_logloss: 0.61076\tvalid_0's auc: 0.771031\n",
      "Training until validation scores don't improve for 5 rounds.\n",
      "[2]\tvalid_0's binary_logloss: 0.556522\tvalid_0's auc: 0.777171\n",
      "[3]\tvalid_0's binary_logloss: 0.519027\tvalid_0's auc: 0.777548\n",
      "[4]\tvalid_0's binary_logloss: 0.492783\tvalid_0's auc: 0.779985\n",
      "[5]\tvalid_0's binary_logloss: 0.47398\tvalid_0's auc: 0.782545\n",
      "[6]\tvalid_0's binary_logloss: 0.460358\tvalid_0's auc: 0.783484\n",
      "[7]\tvalid_0's binary_logloss: 0.451064\tvalid_0's auc: 0.782727\n",
      "[8]\tvalid_0's binary_logloss: 0.444364\tvalid_0's auc: 0.78268\n",
      "[9]\tvalid_0's binary_logloss: 0.439643\tvalid_0's auc: 0.782635\n",
      "[10]\tvalid_0's binary_logloss: 0.435968\tvalid_0's auc: 0.782956\n",
      "[11]\tvalid_0's binary_logloss: 0.433531\tvalid_0's auc: 0.782916\n",
      "Early stopping, best iteration is:\n",
      "[6]\tvalid_0's binary_logloss: 0.460358\tvalid_0's auc: 0.783484\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', colsample_bytree=1.0, learning_rate=0.2,\n",
       "        max_bin=255, max_depth=-1, metric='l1', min_child_samples=10,\n",
       "        min_child_weight=5, min_split_gain=0.0, n_estimators=20, n_jobs=-1,\n",
       "        num_leaves=31, objective='binary', random_state=0, reg_alpha=0.0,\n",
       "        reg_lambda=0.0, silent=True, subsample=1.0,\n",
       "        subsample_for_bin=50000, subsample_freq=1)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "gbm = lgb.LGBMClassifier(objective='binary',\n",
    "                        num_leaves=31,\n",
    "                        metric = 'l1',\n",
    "                        learning_rate=0.2,\n",
    "                        n_estimators=20)\n",
    "gbm.fit(X_train, y_train,\n",
    "        eval_set=[(X_test, y_test)],\n",
    "        eval_metric=['auc', 'binary_logloss'],\n",
    "early_stopping_rounds=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAEWCAYAAAAHC8LZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXmYFcXZvu9H3EA2FSQKCi6oRESiJi6JZpAAxh0l6sQo\nfOAXNTEu+VAxi6JZQI3GNSZRFFyiRo1gNDExyvAzUaNC2NwXJhEXCLgBsgi8vz+qztBzODNzembO\nnHNm3vu6+pru6urqp1uZmq6q531lZjiO4zhOubFJsQU4juM4TmPwDsxxHMcpS7wDcxzHccoS78Ac\nx3GcssQ7MMdxHKcs8Q7McRzHKUu8A3OcVoikX0v6cbF1OE4hkfvAHGcDkqqBHsC6RPHuZvZuE9qs\nAO4ys15NU1eeSJoMLDSzHxVbi9O68C8wx9mYo82sY2JrdOfVHEjatJj3bwqS2hVbg9N68Q7McfJE\n0oGSnpb0kaQ58csqc+5/JL0saZmktySdEcu3Av4M7CBpedx2kDRZ0k8T11dIWpg4rpZ0kaS5wApJ\nm8brHpT0X0kLJJ1Tj9aa9jNtS7pQ0mJJ70k6TtIRkl6T9IGkHySuHS/pAUn3xeeZJWmfxPl+kqri\ne3hR0jFZ971Z0p8krQDGAKcAF8Zn/2OsN07Sm7H9lyQNT7QxStLfJf1C0ofxWb+eOL+NpNslvRvP\nT02cO0rS7KjtaUkD8v4P7JQd3oE5Th5I6gk8CvwU2AYYCzwoqXusshg4CugM/A/wS0n7mtkK4OvA\nu434oqsEjgS6AuuBPwJzgJ7AYOA8ScPybOtzwJbx2kuAW4BvAfsBhwCXSNolUf9Y4P74rL8Dpkra\nTNJmUcdfge2A7wF3S9ojce03gZ8BnYA7gLuBK+OzHx3rvBnv2wW4DLhL0vaJNg4AXgW6AVcCkyQp\nnrsT6ADsFTX8EkDSvsBtwBnAtsBvgIclbZHnO3LKDO/AHGdjpsa/4D9K/HX/LeBPZvYnM1tvZo8D\nLwBHAJjZo2b2pgVmEH7BH9JEHdeb2dtmthL4ItDdzC43szVm9hahEzo5z7Y+A35mZp8B9xI6huvM\nbJmZvQi8CCS/Vmaa2QOx/jWEzu/AuHUEJkYdTwKPEDrbDNPM7B/xPa3KJcbM7jezd2Od+4DXgS8l\nqvzbzG4xs3XAFGB7oEfs5L4OnGlmH5rZZ/F9A/wv8Bsz+6eZrTOzKcDqqNlphZTt2LrjFJDjzOxv\nWWW9gW9IOjpRthkwHSAOcV0K7E74w7ADMK+JOt7Ouv8Okj5KlLUDnsqzraWxMwBYGX8uSpxfSeiY\nNrq3ma2Pw5s7ZM6Z2fpE3X8Tvuxy6c6JpNOA7wN9YlFHQqea4f3E/T+NH18dCV+EH5jZhzma7Q2M\nlPS9RNnmCd1OK8M7MMfJj7eBO83sf7NPxCGqB4HTCF8fn8Uvt8yQV66lvisInVyGz+Wok7zubWCB\nmfVtjPhGsGNmR9ImQC8gM/S5o6RNEp3YTsBriWuzn7fWsaTehK/HwcAzZrZO0mw2vK/6eBvYRlJX\nM/sox7mfmdnP8mjHaQX4EKLj5MddwNGShklqJ2nLuDiiF+Gv/C2A/wJr49fY0MS1i4BtJXVJlM0G\njogLEj4HnNfA/Z8DPokLO9pHDf0lfbHZnrA2+0k6Pq6API8wFPcs8E9C53thnBOrAI4mDEvWxSIg\nOb+2FaFT+y+EBTBA/3xEmdl7hEUxv5K0ddRwaDx9C3CmpAMU2ErSkZI65fnMTpnhHZjj5IGZvU1Y\n2PADwi/et4ELgE3MbBlwDvB74EPCIoaHE9e+AtwDvBXn1XYgLESYA1QT5svua+D+6wgdxUBgAbAE\nuJWwCKIQTANOIjzPqcDxcb5pDXAMYR5qCfAr4LT4jHUxCfh8Zk7RzF4CrgaeIXRuewP/SKHtVMKc\n3iuExTPnAZjZC4R5sBuj7jeAUSnadcoMNzI7jlMLSeOB3czsW8XW4jj14V9gjuM4TlniHZjjOI5T\nlvgQouM4jlOW+BeY4ziOU5a4D6yAdO3a1Xbbbbdiy9iIFStWsNVWWxVbRk5KVZvrSofrSofrqs3M\nmTOXmFn3BiuamW8F2nbffXcrRaZPn15sCXVSqtpcVzpcVzpcV22AFyyP37E+hOg4juOUJd6BOY7j\nOGWJd2CO4zhOWeIdmOM4jtMk1q1bxxe+8AWOOuooABYsWMABBxxA3759Oemkk1izZk1B7usdmOM4\njtMkrrvuOvr161dzfNFFF3H++efz+uuvs/XWWzNp0qSC3LfFOzBJ62LK7zkxVfnBsbyPpPlxv0LS\nIzmurZK0f4p7XSfpnZgOIlM2SpJJGpwoGx7LRkh6KOp7Q9LHcX+2pIMlnR3LTVK33Hd1HMdpOyxc\nuJBHH32U008/HQgr25988klGjBgBwMiRI5k6dWp9TTSaYnyBrTSzgWa2D3AxMKEQN4md1nBC1PBD\ns07Po3YG2ZMJkcExs+FmNhA4HXgqah1oZk8TImZ/jZDAz3Ecp81z3nnnceWVV7LJJqE7Wbp0KV27\ndmXTTYPNuFevXrzzzjsFuXexjcydCWkPCsEgYD4hTUUlUJU49xRwiKTNCHmcdiPkZ6oXM/sXQMwO\nmxNJ3wa+DdCtW3duuHta49QXkB7tKUldULraXFc6XFc6SlXXzl3aUVVVVef5Z555hs8++4xly5Yx\ne/Zsli5dyt///ndWrlxZc93ixYv59NNP622n0eRjFmvODVhH6CxeAT4G9ovlfYD5cb8CeCTHtVXA\n/nne51ZC3qDOwDvAZrF8FCFf0DXAUcAphFTwk4ERietzaojnqoFuDWlwI3N6SlWb60qH60pHueoa\nN26c9ezZ03r37m09evSw9u3b2ze/+U3bdttt7bPPPjMzs6efftqGDh2a6r6UsJE5M4S4J3A4cIfq\n+6RpBJI2B44ApprZJ4QsskOzqt1LGDo8mZBs0HEcx0nBhAkTWLhwIdXV1dx7770cdthh3H333Qwa\nNIgHHngAgClTpnDssccW5P5FXYVoZs8A3YCGY16l43BCptp5kqqBr1B7zgsze46Qxrybmb3WzPd3\nHMdps1xxxRVcc8017LbbbixdupQxY8YU5D5FnQOTtCfQDlgKdGjGpiuB083snnifrYAFkrLvcTGw\nqhnv6ziO0yapqKigoqICgF122YXnnnuu4PcsxhdY+8zSdMICi5Fmti5HvcGSFia2g2L5o4my+7Mv\nip3UMODRTJmZrQD+DhydrGtmfzaz6fkKl3SOpIVAL2CupFvzvdZxHKe1Uiwjc4t/gZlZuzrKqwlD\nephZFdA+R7WKPNr/FNgmR/nxicPJOc6PyjquovbKRczseuD6hjQ4juO0JTJG5k8++QTYYGQ++eST\nOfPMM5k0aRJnnXVWs9+3qHNgkpbnKBsvaWzcnyzpU0mdEuevSxqJJS2XtHfCcPyBpAVx/2913LeP\npJUJQ/XTkvbIqlOXCfrG5np+x3GccqetGZnT8gZwLNSYkwcRlsUjaRjhS+3ORP1lwAVxpePX6mn3\nTdtgqJ4C/CBzogETtOM4jhNpy0bmfLgHOAm4izCE+A/g6wBm9hdJKy1EzgDCV1sj7pFtqK7PBF0v\nbmRuGqWqzXWlw3Wlo1R1uZG5frPx8hxl44GxcX8yMAJ4FtgauAX4KgkjcXYbZBmS67hvH2AlwVD9\nJvAesFPifL0m6Hyfz43M6SlVba4rHa4rHeWqqy0amRvDHwiG4wMIYaCag8wQ4q7AecBvIW8TtOM4\nTpunTRuZU3Av8BPgcTNbX4D2H2bDXFeDJmjHcRynbtqEkTlfzOw/kn4I5FxV2Ax8hTCUCPmboB3H\ncZxIWzEyJ+mQZVb+fl0Vzew3ZvZmXecbwa6ZZfTAz4HT8zRBj8rS3KsZNTmOA7z99tsMGjSIfv36\nsddee3HdddfVnLvhhhvYY4892GuvvbjwwguLqNIpNkX9AjOzejtQyzIXJ8r7JPY75nNNVp1qchul\noREmaMdxmpdNN92Uq6++mn333Zdly5ax3377MWTIEBYtWsS0adOYO3cuW2yxBYsXLy62VKeIFO0L\nLJGZeb6k+5NDdIkMyXvG4y0lvSJp70SdCyX9uoF7nC9plaQuibKK2PaYRNkXYtlYSTdFXS8lzM6z\nJY2Idb8n6VVJL0q6sjnfieM4ge233559990XgE6dOtGvXz/eeecdbr75ZsaNG8cWW2wBwHbbbVdM\nmU6RKeYQYiatSn9gDXBm4lwlYdjuZAAzW0VYKfgrBXoCZxCC8dbHaGAtYUFGJv7irwgZmU9K1Etm\nZP6uBV/ZEWxYqTjQzB6QNIhgqh5gZnsBv2jKC3Acp2Gqq6v517/+xQEHHMBrr73GU089xQEHHMBX\nv/pVnn/++WLLc4pIqSzieAoYACCpI/Blgpn4YYIvDDN7TNJo4DTgSGC8mdWZzVnSroRI90cCPzCz\nYbG8AhgLdJbUA1hMWHn4pzx0ngVMNLPVUVO94xcrP1tHn3GP1lelKPzf3msZVYK6oHS1ua501Ker\neuKRebezfPlyTjjhBK699lo6d+7M2rVr+fDDD3n22Wd5/vnnOfHEE3nrrbfqzZLutF6K3oFJ2pQQ\nWeOxWHQc8JiZvRbjGu5rZrPiufOA54DXzezOHM0lqSRE8XgK2EPSdlkdzgPAN4B/AbOA1XnI3R04\nRNLPCGlYxppZrT8Bk5E4unfvzu8P3yqPZluW5cuXM7kEdUHpanNd6ahPV74RGdauXcvFF1/MAQcc\nwDbbbENVVRUdOnRgl112YcaMGQCsWbOGadOm0bVr17x1FSQiRBNxXY0kH7dzITZgHSESxmzgBmDz\nWP4oMCTunwNclXXdHcCJebQ/H+gb968Bvhv3K4BHgM8BM4AbCUbl8cQIILYhWsf8HG1eDwj4ErAA\nUF0aPBJHekpVm+tKR1N1rV+/3k499VQ799xza5XffPPN9uMf/9jMzF599VXr1auXrV+/vsV0FQrX\nVRvyjMRRzC+wWjEMASRtCxwG9JdkhCFAk3RhfCiA9XGrE0kDgL7A43FoYXPgLeCmTB0ze1/SZ8AQ\n4Fzg4Dw0LwT+ELU8J2k9IaP0f/O41nGcPPnHP/7BnXfeyd57783AgeHXxM9//nNGjx7N6NGj6d+/\nP5tvvjlTpkzx4cM2TNGHELMYAdxhZmdkCiTNIBiN04SQqiTMkU1ItLNAUu+sepcA25nZujz/EUwl\ndLBVknYndIxLUuhyHCcPvvKVr7Dhb9ba3HXXXS2sxilVim1kzqYSeCir7EHgmynbOTlHOw/F8hrM\n7GkzS5Oo5jZgF0nzCeGtRlpd/8ocp5VTl9n4xz/+MWPGjGHgwIEMHTqUd999t8hKndZK0b7ALMuA\nHMsqcpRdn3U8Ko+2d85RlozyUSVpOCFIcD8ze8XMxkvqK+kRoB/wEfBfSYea2f+TNAq4ihCdfi2h\n83+/IS2O01qpy2x8wQUXMHjwYCoqKrj++uu5/PLL+fWv67VsOk6jKLUvsJakltdM0paEBSS/NbNd\nzWw/4HvALolr7rMNvrCBZvZSi6t2nBKhLrNx586da+qsWLHC56icglFqc2CpiJE5spfTrzazAxq4\nLpfX7BTgGTN7OFPPzOYTVh46jlMPSbMxwK233sqpp55Kly5dmD59epHVOa2Vsu7AzGweMLDBihuz\nkdcM2IvgB6uPkyR9JXF8kJmtrKuyG5nTU6ra2pquppiNAU4//XTuuusuJkyYwI033shll13W7Bod\nR21xDYKkR4FrzexxSecAOxKW7P/bzK6LdR4iLMV/zcyOj3Ng+5vZ2Q20XWNk7tat+36XXHtLAZ+k\ncfRoD4vq7HaLS6lqa2u69u7ZpeFKbDAbf/GLX+TEE0+sKV++fDkdO3bk/fff5+KLL+b2229vfpGN\nIKOr1HBdtRk0aNBMM9u/wYr5mMVa0wZsC6wE/g1UA28D/wHGAFOy6u4PVMX9UcCNae7lRub0lKo2\n17UxdZmNX3vttRpd119/vZ1wwglFUJcb/++YDjcylx51ec1eAy6WdIxtmAfzJJaOUwd1mY0nTZrE\nrFmz6NSpE7179/YViE7BaIsdWCUwMass4zU7CrhG0rXAImAZ8NNEvew5sO+Y2dOFFOs4pUpdZuMj\njjiCqqqqmuy8jlMo2lwHZg17zY6o47rJeDJLx3GckqEt+8Acp9VTV7SMDz74gCFDhtC3b1+GDBnC\nhx/WmZnIcUoWz8hM/hmZJV0VdcyV9JCk/HI4OE6RyETLePnll3n22We56aabeOmll5g4cSKDBw/m\n9ddfZ/DgwUycmD2q7jilT2vPyFwJPA8MzypvVEZm4HGgv5kNIC76SPvQjtOS1BUtY9q0aYwcORKA\nkSNHMnVqmpCgjlMalMocWKEyMncELgB+QO35q//QiIzMZvbXxOGzhBWNdeJG5vSUqrZS1ZUmmWUy\nWsaiRYvYfvvtgdDJLV5cb3JxxylJit6BlVlG5iSjgftyPE/SyMwle69N2Wzh6dE+/EIuRUpVW6nq\nyjdj7sqVKzn33HM5/fTTmTVrFmvXrq11XfZxS+lqaVxXOkpVVw35mMUKsVGGGZkT535ISM9SZzZm\ncyNzoyhVbeWsa82aNTZ06FC7+uqra8p23313e/fdd83M7N1337Xm/n+1nN9XMXBdtSFPI3MpzIEN\nNLPvmdmaREbmWyVVE4b/TlLtcNZpMzJXE+a4KpN1zOx9IJOR+Yl8RUsaSfCLnRJftOOULGbGmDFj\n6NevH9///oaMQscccwxTpkwBYMqUKRx77LHFkug4jaboQ4hZlHRGZkmHAxcBXzWzT1PocZyiUFe0\njHHjxnHiiScyadIkdtppJ+6///4iK3Wc9JRaB1ZflIw0HdjJhHm1JJmMzP/MFFj6KBo3AlsQvuwA\nnjWzM+u/xHGKR13RMgCeeCLvgQfHKUnabEbmHOfHZx1XA/2zynZr6N6O09yMHj2aRx55hO222475\n80N6utmzZ3PmmWeyZMkSunbtyq9+9Su+9KUvFVmp47QsrSoSR6HN0ZJGSno9biML+zSOExg1ahSP\nPfZYrbILL7yQSy+9lFtvvZXLL7+cCy+8sEjqHKd4lHUHJmnvRKSM2bF4tRXAHC1pG+BS4ADgS8Cl\nkrYuxHM5TpJDDz2UbbbZplaZJD755BMAPv74Y3bYYYdiSHOcolJqc2CpsKyMzJKWm9kB8bC5zdHD\ngMfN7IPY5uMEA/Q9delzI3N6SlVbKWQ+TnLttdcybNgwVq1axWabbcbTT3tSBKftUdYdWF0UyBzd\nk5D8MsPCWJZ9bzcyN4FS1VYoXfmaRN9//31WrFhRU//6669nzJgx7Lfffjz//PMcf/zxXH311c2u\nr7GUqgHWdaWjVHXVkI9ZrFw2CmiOJnjSfpQ4/jHwf/Vd40bm9JSqtmLrWrBgge211141x507d7b1\n69fb9OnTbf369dapU6ciqtuYYr+vunBd6Sh1I3Nr+wJbaSEQbw0Jc3R/SQa0A0zShfFFQR7maMIX\nV0XiuBc5VjM6Tkuwww47MGPGDACefPJJ+vbtW2RFjtPytLYOLBfNZY7+C/DzxMKNoXg0eqcFqKys\npKqqiiVLltCrVy8uu+wybrnlFs4991w++ugjunXrxm9/+9tiy3ScFqctdGDNYo42sw8k/YSQngXg\ncosLOhynkNxzT+51QjNnzqSqqoqKioqWFeQ4JUJZL6PPxuowR5vZY1ll15vZWYnjURbyfTXU/m1m\ntlvcbm8e1U5bYPTo0Wy33Xb071/LG88NN9zAHnvswV577eVeLsdJSavqwBynVMllRp4+fTrTpk1j\n7ty5vPjii4wdO7ZI6hynPClaB1boqBmxzvmSVknqkiiriG2PSZR9IZaNlXSPpJXxuvVxf4WkEZJ+\nImlu1P1XSe4edfIilxn55ptvZty4cWyxxRYAbLfddsWQ5jhlSzHnwGpWDEq6mxA145p4Lhk1Y7yZ\nrZKUiZpxKLADIWrG/g3co5IwZzWc2hmZ5wEnAZPi8cnAHAAzq4ya+gCPWIjqQSz7q5n9OO6fQ4hm\nX2cwXzcyp6dUtdWnq7Fm5Ndee42nnnqKH/7wh2y55Zb84he/4Itf/GJTZDpOm6JUFnE0d9QMJO0K\ndCT4t35A7Q7sP0BnST2AxYSIGn9qSKSZfZI43ArYKMy3G5mbRqlqq09XY83IH3/8MfPmzWPixIm8\n8sorHHPMMfzud78jn9Q+GUrVaOq60uG6Gkk+ZrFCbMDy+HNTYBpwVjz+FjAp7j8N7Ju4ZgeCH2t6\nHu3/iGA23gSoJuT9gg0Zmc8BziZ0lreTZ0Zm4GeEiBzzge71aXAjc3pKVVtz6Mo2Iw8bNqxWu7vs\nsostXry4xXUVAteVDtdVG8ogI3P7GID3BcIXUWY4rxK4N+7fSyKTspm9CzwJ3JxH+ycD95rZeuAP\nwDeyzv8+llVSTzzDbMzsh2a2I3A3oQN0nEZx3HHH8eSTTwJhOHHNmjV069atyKocp3woiTmwDM0V\nNUPSAKAvGxJPbg68BdyUqWNm70v6DBgCnAscnFL/7wghqi5NeZ3TBsllRh49ejSjR4+mf//+bL75\n5kyZMiXV8KHjtHVKZQ4sQ3NFzagkzJFNSLSzQFLvrHqXEIYW1+Xzi0NSXzN7PR4eA7ySQpPThqnL\njHzXXXe1sBLHaT2Umg+sEngoqywTNSMNJ+do56FYXoOZPW1mU1O0OzEu+59LCCV1bkpdTisll1F5\n/Pjx9OzZk4EDBzJw4ED+9KcG1wk5jpOCon2BWR1RM3KUXZ91PCqPtnfOUfb9xGFVjvPjs46rgf5Z\nZSc0dG+nbTJq1CjOPvtsTjvttFrl559/vhuUHadAuJGZjYzMN0VdL0UTcybj84hE/bGxvs+4O0Bu\no7LjOIWlJBZxNMHIfFpcyZhktW3IytwYI/N3o6Y+BCNz9kKTHQkLP/7T4AO6kTk1papt8uFbNeq6\nG2+8kTvuuIP999+fq6++mq233rrhixzHyQttWNzXwjeWlmeGESWdCQwws+9EI/OrRCOzme2ZuOb3\nhJV/RwJ/tHoyKEcj8x+Bs4AfmNmwWF4BjAU6E5bRLyYkwPwTsNTMfhHr9SErEkcsfwD4CcG7tr+Z\nLck6nzQy73fJtbekfjeFpkd7WLSy2CpyU6radu7Sjo4dNxr1rsX777/PxRdfzO23hzjPH3zwAV26\ndEESt912G0uXLuWiiy5qVl3Lly9vUFcxcF3pcF21GTRo0EwzayjSkhuZSWFkJqw8vC7uVwPd6tPg\nRub0lKq2fHRlG5XzPdcUyvl9FQPXlQ43MtdNWRmZ4xzdDwlL7x2nQd57772a/YceemijVCqO4zSN\nkpgDy1DiRuZdgZ2BObHNXsAsSV8ys/fzeF6nFZPLqFxVVcXs2bORRJ8+ffjNb35TbJmO06pwI3Oe\nRmYzmwfU5LuQVE2OOTCnbZLLqDxmzJgcNR3HaS7cyJzOyOw4OXEjs+O0PG5k3nB+fNZxNVlG5qzz\nfRrS4bQd3MjsOC2PG5nJ38gs6RuSXlTI1NzwEk+nzeBGZsdpeUpiEUe5GJkl9QOOB/KajXcjc3pK\nVZsbmR2n9HAjc0ojczxXRfCMvZDjnBuZm0CpanMjczpcVzpcV23cyFygjMzxXBVhBWK9GtzInJ5S\n1eZG5nS4rnS4rtrgRubCZGR2nHxxI7PjFJaSmAPLUOJGZsepEzcyO07L40bmFBmZHacu3MjsOC2P\nG5lTGJnj8v6FwEHAo5L+klKXU8bkMitn+MUvfoEklizxwCyO01K4kXnD+fFZx9VsnJH5ITbuGJ02\nQl1m5bfffpvHH3+cnXbaqUjKHKdt0uJfYAkD8xxJsyQdHMv7SJof9yskPZLj2qo0BmJJ10l6R9Im\nibJR0bQ8OFGWMU6PkPRQ1PeGpI8TRuaDJd0t6dVovr5N0mZNextOOVGXWfn888/nyiuvxIehHadl\nSf0FJmlrYEczm9vIeyYNzMOACcBXG9NQjMyR7QVbbWYHxE5rOPA2cCi1v7rmEYYrn4jHSSPz8Nh2\nBWFZ/VGJ+3UlLPMH+B1wOvWsiHQjc3qKpa164pGNuu7hhx+mZ8+e7LPPPs2syHGchsirA4vG3WNi\n/dnAfyXNyBqWawydgQ8be7GFCPED6zg9CJgP3EforKoS554CDolfUFsAuxGeq6H71URjlfQcIaVK\nLbKMzFyy99p8HqVF6dE+dBSlSLG0VVVV1Xt++fLlVFVV8f7777NixQqqqqpYtWoVF110EVdddVXN\n8T/+8Q+6dOlSb1vNSUZXqeG60uG6Gkk+ZjHgX/Hn6cBlcX9uPtfmaGsdobN4BfgY2M+yjMNEs3GO\na6vIw0Ac694KnEroJN8BNovlo4AbCWGrjgJOAS4lhJoakbg+p4Z4bjNgFnBIfRrcyJyeUtWW0ZU0\nJM+dO9e6d+9uvXv3tt69e1u7du1sxx13tPfee6/FdZUarisdrqs2NLOReVNJ2wMnEqJYNIWVZjbQ\nQoiow4E71MyTB5I2B44ApprZJ8A/gaFZ1e4lDB2eTHoj86+A/2dmaZb2O62Mvffem8WLF1NdXU11\ndTW9evVi1qxZfO5znyu2NMdpE+TbgV0O/AV408yel7QL8HpTb25mzwDdgO5NbSuLw4EuwLyYePIr\nJCJ6xHs/R1hl2M3MXsu3YUmXEvQ2dfjUKTMqKys56KCDePXVV+nVqxeTJk1q+CLHcQpGXnNgZnY/\ncH/i+C3ghKbePKZLaQcsBTo0UD0NlcDpZnZPvM9WwIJkypbIxcCqfBuVdDowDBhsIUSV04bIZVZO\nUl1d3TJCHMcB8vwCk7S7pCcSy9wHSPpRI+/ZPrM0nbDAYqSZrctRb7CkhYntoFj+aKLs/uyLYic1\njBC1HgAzW0FIz3J0sq6Z/dnMpqfQ/mugB/BMfIZLUlzrlDluZHac0iLfZfS3ABcQ82CZ2VxJvwN+\nmvaGZtaujvJqonHYzKqA9jmqVeTR/qfARmYdMzs+cTg5x/lRWcdVZBmezazUQm85LYgbmR2ntMh3\nDqxDnDNKUnLrsFXgLM+J9mdLeriwT+OUGm5kdpzSIt8viiUxQaQBSBoBvFf/JYUnGqGvyCpeYGbD\nm5Dlub4YknRyAAAgAElEQVRIHxtF0K8PNzKnx43MjuPkS14ZmeOqw98SUo58CCwATjGzfxdWXjpa\nIMtzTfv11PGMzE2gWNr27lm/+TiTmTaZdXnVqlWcf/75XHXVVXTs2JGTTz6Z3/zmNy1uZPZMvvnj\nutJR9hmZCcOMJ8b9rYBO+RjMirFR+CzPawkJOJ8FjmuovhuZ01Oq2tzInA7XlQ7XVRvyNDI3OIRo\nZuslnQ383sJqvlImk+UZQrioZJbna+N+JsvzLAhZniU9SX4G7Z1i/V2AJyXNM7M3m0++U05kjMwZ\n+vTpwwsvvEC3bt2KqMpx2g75LuJ4XNJYSTtK2iazFVRZ48hE+RhoZt8zszXakOX51mhqvgA4KSv6\nR4NZniF0dvHnW4QVil9o7gdwShc3MjtOaZHvIo7R8ed3E2UG7NK8cgpCs2R5jlH4PzWz1ZK6AV8G\nrmxusU7p4kZmxykt8voCM7Odc2zl0HlB82V57ge8IGkOMB2YaGYvNYM+p4XIZUS+4IIL2HPPPRkw\nYADDhw/no48+KqJCx3HSkG8kjtNybYUWlxarI8uzmT2WVXa9mZ2VOB5lZg800PbTZra3me0Tf/r4\nUZkxatQoHnus1v8KDBkyhPnz5zN37lx23313JkyYUCR1juOkJd85sC8mtkOA8YT8YI2m0KbjWOd8\nSaskdUmUVcS2xyTKvhDLxkq6Kep6SdLKhHF5hKTxChmeM2VHNOUdOC1LLiPy0KFD2XTTMJJ+4IEH\nsnDhwmJIcxynEeQbzPd7yePYIdTpl8qTZGbmQpiOM+08T8jMPDlRPg84iQ2rFGsyMhPiHX45q53V\nZvaApP7AL83sF3k9oBuZU9MUbY01I2e47bbbOOmkk5rUhuM4LUdjY/t9CvRtRh1PAQMAoun4y0TT\nMeFrDzN7TNJo4DSC6Xi8mdWZzTlGDulIWHX4A2p3YP8BOkvqASwmpF/5U7zPPGCgpD6EhJZ5R96I\n9/WMzE2gKdryyRybzKic5K677uKjjz6iZ8+eOdsp1cy0risdrisdpaqrhnzMYsAfCZ3JwwS/1FvA\nFflcW0+bhTYd/wj4MWGYtBrYLpZXxGc4Bzib0FneTugoxyau70PMEJ0oGx/bmgvcBmxdnwY3Mqen\n0NqSRuQMkydPtgMPPNBWrFhRNF2NxXWlw3Wlo+yNzJHkkNla4N9m1tTJgkKbjk8GhlswYv8B+AZw\nU+L87wnpXPYkZGQ+OI82bwZ+QrAQ/AS4mg0WA6cMeeyxx7jiiiuYMWMGHTo0Z0o6x3EKTb4d2BFm\ndlGyQNIV2WUp2SgwbsJ03F+SEZJdmqQLY68MeZiOJQ0gDHE+Hv3KmxO+Gms6MDN7X9JnwBDgXPLo\nwMxsUeIet5BfR+qUCJWVlVRVVbFkyRJ69erFZZddxoQJE1i9ejVDhgwBwkKOX/+63rVBjuOUCPl2\nYEOA7M7q6znKmkqzmI4JX23jzaxmTbSkBZJ6Z9W7hDC0uC6fVBiStjezTBT+4cD8FJqcIpPLiDxm\nzJgcNR3HKQfqXUYv6SxJ84A9JM1NbAsI80DNTXOZjk/O0c5DsbwGC96uqSnavVLSPElzCYtMzk+p\nyykibmR2nNZFQz6w3wFHExZvHJ3Y9jOzbzXlxlZY0/HOZvZKVtn3zewKM6sys6NyXDPeEsvjzaza\nzPpn1TnVgol5gJkdk/gac8oANzI7Tuui3g7MzD6Ov8grLeT+WklYwNBRUqPypycMzHMkzZJ0cCzv\nI2l+3K+QtNH8kqQqSQ3niNlQ/7poPN4kUTYqmpYHJ8oyxukRkh6K+t6Q9HHCtHywAj+T9JqklyWd\n05h34BQHNzI7TusirzkwSUcTTMY7EHxTvYGXgb0acc+kgXkYMAH4aiPaIUbmyDZUrzazA2KnNRx4\nGziUED0+wzzCcOUT8bjGyGxmw2PbFYRl9TVfa5L+B9gR2DOubtyuPn1uZE6PG5kdx8mXfBdx/BQ4\nEPibmX1B0iBCB9BUOhMyPDcKi6bjOk4PIiyyuI+gtSpx7ingEEmbAVsAuwGzsxvIwVnAN81sfbz/\n4uwKbmRuGm5kTofrSofrSkep6qohH7MY0VRG+ErZJO4/l8+1OdpaR+gsXgE+JsynQcI4TDQb57i2\nCtg/z/vcCpxK6CTfATaL5aOAGwlflEcBpwCXEiJ1jEhcv5EGYCnwQ0JW5j8DfevT4Ebm9LiROR2u\nKx2uKx2lbmTON5jvRzHE01PA3ZKuIxiaG0Mm6eSehBBOdyifNewpkLQ5cAQw1cw+Af4JDM2qdi9h\n6PBkgpE5H7YAVpnZ/sAthGgcThmTMTI//PDDbmR2nDIj3w7sWEL8w/OAx4A3CasRm4SZPQN0A7o3\nta0sDge6APMUsjB/hawhTzN7DugPdDOz1/JsdyFhWT+EZfkDmkWt0yLkyqh89tlns2zZMoYMGcLA\ngQM588wziy3TcZw8yTca/YpoAu5rZlNi6pN2Tb15TJfSjjA015x//lYCp5vZPfE+WwELkilbIhcD\nq1K0O5UQKeQ2wsKTfDs+pwRwI7PjtC7yTWj5v8ADwG9iUU/CL/PG0D6zNJ2wwGKkma3LUW+wpIWJ\n7aBY/mii7P4cWjsAw4CapWxmtoKQnqXWV6OZ/dnMpqfQPhE4IZq7JwCnp7jWKTJuZHac1kW+Q4jf\nJURt/wTAzF4H6l1CXhdm1i7OgQ20kN340VheYxy2YDZub2a9EtszFozOPRJl38jR/qdmtk2c+0qW\nH29m95nZZDM7O8d1tQzSlsPwbGYfmdmRFszMB5nZnOx2nNLFjcyO07rItwNbbWZrMgeSNiUYmlNT\nzkbmRP0bJC1vzPM7xcONzI7TusjXBzZD0g8Iw39DgO8QcoQ1huY0Mg8DrsgqXmBmwwthZI7l+wNd\n89HnRub0uJHZcZx8ybcDGweMIfziP4OQvfjWZrh/U43MfwH+UsfpZjcyS2oHXEUILjy8jjpuZG4C\nbmROh+tKh+tKR6nqqqE+kxiwUz5msjQb5W1kPhc4P+4vb0iDG5nT40bmdLiudLiudJS7kblmpaGk\nB+urmIKyNDJL2oGQ1fmG5tTqFBc3MjtO+dJQB5bsWHZp7ptbeRmZv0AYanwjttlB0hvNKdopLG5k\ndpzWRUNzYFbHfrNQTkZmC8v9P5c5lrTczHZrJr1OC+BGZsdpXTT0BbaPpE8kLQMGxP1PJC2T9EkD\n19ZFORuZnTLGjcyO07poKKFlOzPrbGadzGzTuJ857tyYG1oZG5mz6m+UUdopbdzI7Diti3yNzM1G\nORuZJU2KuudKeiBG6HfKBDcyO07rIl8fWHNStkZmSfMzX3aSrgHOJsRHzP2gbmROjRuZHcfJl2J0\nYEnKysic6LwEtCfHwhY3MjcNNzKnw3Wlw3Wlo1R11ZCPWaw5N8rYyBzLbwcWAdOBDvVpcCNzetzI\nnA7XlQ7XlY5yNzIXgrI0Mmcws/8BdgBeBny8qcxxI7PjlC/F6MBqsPIyMievXUcYmjyhWdQ6LYIb\nmR2ndVHUObByMjLHr8RdzeyNuH80YRjUKRPcyOw4rYtifIGVq5FZwBSFbMzzgO2By/O81ikB3Mjs\nOK2LFv8CM7N2dZRXE4b0MLMqwiq/bCryaP9TYJsc5ccnDifnOD8q67iKxMpFM1tPyErtlCmjRo3i\n7LPP5rTTTqspGzJkCBMmTGDTTTfloosuYsKECVxxRbYzw3GcUqSoc2DNTcIkPV/S/ckhw4RZec94\nvKWkVyTtnahzoaRf19F2b0kzY/svSvLJkjLDjcyO07ootg+sSTRgZL4bOJOwXB7CvNjfCasOx5vZ\nKknnAb+SdChhZeEZQF2RPt4DDjaz1TECx3xJD5vZu3XpcyNzetzI7DhOvigsuW8dxAjxHeP+mcAA\nM/tO7HBeJZibH45L+DPX/J4wX3Yk8EczuzOP+2wL/As4MLsDyzIy73fJtbc0z8M1Iz3aw6KVxVaR\nm6Zo27tnlwbrvP/++1x88cXcfvvttcrvuusuXn31VS6//HJyuTqWL19Ox46lFznMdaXDdaWjWLoG\nDRo008waDhuYj1msXDZilmTCl+U04Kx4/C1gUtx/Gtg3cc0OwEJgeh7t7wjMBT4FvttQfTcyp8eN\nzOlwXelwXelwI3PL0j6ubnwB+A8wKZZXEozLxJ81njALX1BPAjc31LiZvW1mAwihp0ZK6tGM2p0i\n4EZmxylfynoOLAc1gYIzxOG+w4D+kozgOzNJF8aeHmB93PLCzN6V9CJwCPBAQ/Wd0qCyspKqqiqW\nLFlCr169uOyyy5gwYQKrV69myJAhQFjI8etf51zH4zhOidHaOrBcjADuMLMzMgWSZhCiczyVbyOS\negFLzWylpK0JS+qvaeAyp4RwI7PjtC5a2xBiLiqBh7LKHgS+mbKdfsA/Jc0BZgC/MLN5zaDPaSHc\nyOw4rYtW1YFZjizJFrI4P5ZVdr2ZnZU4HmWJbMx1tP24mQ2wkEV6gJn9tvmUOy2BZ2R2nNZFq+rA\nCmxkHijpmWhinivJDUNlhhuZHad10drmwJLZnhtrZD4trmRMspqQW+w0M3td0g7ATEl/MbM6x5zc\nyJweNzI7jpMvbmRuhJE5XjeHkADz9axyNzI3ATcyp8N1pcN1pcONzK3IyJy45kuEhJab1FfPjczp\ncSNzOlxXOlxXOkrdyNzahhDbJ4b/nqK2kfnauJ8xMs+CGk/Xk8Aj+dxA0vbAnYQ0MHl7x5zSJGNk\nnjFjhhuZHafMaG0dWEGNzJI6E4Ybf2RmzzavdKfQuJHZcVoXra0Dy0VzGZk3J/jJ7jCzjRJpOqWP\nG5kdp3XRqpbR10FzGZlPBA4FRmUySksa2NBFTsuSy6x8//33s9dee7HJJpvwwgsvFFGd4zjNSavq\nwKywRua7zGwzMxuY2LKX2ztFJpdZuX///vzhD3/g0EMPLZIqx3EKQYt3YAmz8RxJsyQdHMv7SJof\n9yskbbSoQlKVpIaXVm6of52kdyRtkigbFQ3NgxNlGZPzCEkPRX1vSPo48bV1sKTJkhb4F1jpksus\n3K9fP/bYY48iKXIcp1AU4wtsZfx62Qe4GChI7J7YaQ0H3iYM/SWZRyKlCsHcPCfuX5KjudVm9nTc\nv8C/wBzHcYpPsRdxdAY+LFDbg4D5wH2Ezqoqce4p4BBJmwFbEPJ7zQawEKB3oKQKYKyZHZXmpllG\nZm64e1rTnqIA9GhPSeqC+rXlY1SGYFZesWIFVVVVtco/+ugjZs6cyfLly1PrWr58+UbtlQKuKx2u\nKx2lqitDMTqwjFdrS2B7whL3QlAJ3EMwNP9c0mZm9lk8Z8DfgGFAF+BhYOc82/2ZpEuAJ4BxZrY6\nedJCkN/fAuyxxx72vVOObfKDNDdVVVWcWFFRbBk5aQ5t1dXVbLXVVlRktdO1a1f2228/9t8/71Ho\nWrqy2ysFXFc6XFc6SlVXhmIOIe4JHA7coVyxe5pAXPJ+BDDVzD4B/gkMzap2L2Ho8GRCR5cPFwN7\nAl8EtgEuahbBjuM4TmqKugrRzJ4BugHdm7npwwlfVvMkVRM8X8k5L8zsOaA/0M3MXsunUTN7L0Y6\nWQ3cTggp5ZQQlZWVHHTQQbz66qv06tWLSZMm8dBDD9GrVy+eeeYZjjzySIYNG1ZsmY7jNANFnQOL\nqU3aAUuB5ozjUwmcbmb3xPtsBSxIpleJXAysyrdRSdub2Xvxi/E4whybU0LkMisDDB8+vIWVOI5T\naIrxBdY+swydsMBipJmty1FvsKSFie2gWP5oomyjiBixkxpGCPkEgJmtIKRSOTpZ18z+bGbTU2i/\nW9I8wirGbsBPU1zrNAO5jMoffPABQ4YMoW/fvgwZMoQPPyzUuiDHcUqJFu/AzKxdYhn6Pmb2aCyv\nNrP+cb/KzNqbWa/E9kw0JfdIlH0jR/ufmtk2ce4rWX68md1nZpPN7Owc19UyM0cNR2XVOczM9jaz\n/mb2LTNLv5zNaRK5jMoTJ05k8ODBvP766wwePJiJEycWSZ3jOC1Jq4rEUciMzPH8TpL+KullSS9J\n6lPI53E2JpdRedq0aYwcORKAkSNHMnXq1GJIcxynhSnrDkzSsERUjIypeEH8kltDyMicIZmRGTNb\nBWQyMktST0JG5ovrueUdwFVm1o+wgGNx8z6R0xgWLVrE9ttvD8D222/P4sX+n8Vx2gLFNjI3CTP7\nC/CXzHHMyJyZrX8KGBDLOwJfJmZkBsbH6x+TNBo4jZCRebyZ5ZxAkfR5YFMzezxe2+Dw4crP1tFn\n3KMNVWtx/m/vtYwqQV0Akw/fqtgSHMcpE8q6A6sLSZsCXwcykyXHAY+Z2WuSPpC0r5nNiufOA54D\nXjezO+tpdnfgI0l/IJie/0YwMtdagJKMxNG9e3d+X4K/kJcvX16yHUU+zv/sSBudO3fmwQcfZNtt\nt2Xp0qV06tSp2aMHlGpEAteVDteVjlLVVUM+aZvLZQPWEUJCzQZuADaP5Y8CQ+L+OYRhwOR1dwAn\nNtD2COBjYBdCx/8gMKa+a3bffXcrRUo1fblZftoWLFhge+21V83x2LFjbcKECWZmNmHCBLvggguK\noqsYuK50uK50FEsX8ILl8Tu/tX2BFTIj80LgX2b2Vmx3KnAgMKk5H8Cpn1xZlceNG8eJJ57IpEmT\n2Gmnnbj/fs836jhtgdbWgeWiWTIyA88DW0vqbmb/JXSKnh2xhanLqPzEE0+0sBLHcYpNWa9CzJNm\nychsYa5rLPBENDMLuKVZFDp540Zmx3EytKoOzAqYkTnWe9zMBlgwM48yszXNo9zJFzcyO46ToVV1\nYIU2Msc6nWOW5xsL9yROXbiR2XGcDK2qA2NDqpamGJnvTJqj4/bPRDs/AWa0yNM4eeFGZsdpm7Tm\nRRyNNTL/AxiYq0FJ+wE9CP6yBrMiupE5PaXqT3Mcp/RolR1YIYzMkjYBrgZOBQbXU8+NzE3Ajczp\ncF3pcF3pKFVdNeRjFiuXjcIamc8GLoz7o4AbG9LjRub0uJE5Ha4rHa4rHW5kblkKaWQ+CDhE0neA\njsDmMfbiuOZ9BKc+3MjsOE6G1taB5aJZjMxmdkri+lHA/t55tTxuZHYcJ0NrW4WYi2YxMjv58+qr\nrzJw4MCarXPnzlx77bXFluU4TiujYB1YwpOV2cbF8ipJr0qaG31YN0rqGs/1kTQ/q53xksYmjsfG\n6+ZLmiPptET1nSV9JumMRFl7YKKk/0j6b8wbNhq4QlK1pG5mNgp4VtI0Sa9LelPSdZI2j/esiB6y\nowHMbDLQR1JFc7+31sAee+zB7NmzmT17NjNnzqRDhw4MHz684Qsdx3FSUMgvsIwnK7MlwyOcYmYD\nCMvcVwPT8mlQ0pnAEOBLFrxehxJCOmX4BvAs4asLADM7IM6LXQLcl9BTnWhXwB+AqWbWl5A6pSPw\ns0TbC4Ef5vnsTuSJJ55g1113pXfv3sWW4jhOK6Ooc2BmtkbShcAbkvYhpCupjx8Ag8zsk3j9x8CU\nxPlK4P+A30nqaWbv5CnlMGCVmd0eI3PcSejcPy9pKLAZMAfYTNIQi0ktnYa59957qaysbLii4zhO\nSgrZgbWPw3UZJpjZfdmVzGydpDnAnsA/s89nkNQJ6GRmb9Zxfkfgc2b2nKTfAycB1+SpdS9gZtQz\nj2hklvQvYCSwDSGQ70/jllcH1lqNzNUTj8yr3po1a3j44YeZMGFCo+/lOI5TF4XswDZa0l4PmWFA\nq+O8xTp1nYcQIur3cf9eQp6ufDuwutquVW5mT0lC0iF1NpQwMnfr1p1L9l6bp4SWo0f70Ik1lnyN\njX//+9/Zeeedefnll3n55ZfzuqZUjZOuKx2uKx2uq5HkYxZrzAYsr6O8irAEPXPcDniLMB/WEXgn\nq/71wMi4/zawSx3tzgLeAarjtgbomzg/iizzcazXDfga8P+yznUGlgIdgArgkVg+lBDh4xGgor53\n0NaNzCeddJLddtttqa5xQ2c6XFc6XFc6St3IXNRl9JI2AyYAb5vZXDNbDrwnaXA8vw1wOCEIL7Hu\nTZI6x/OdJX1b0h7AVmbW08z6mFmfWPfkPKU8AXTIrGiU1I4QNmqymX2arGhmfwW2BvZp9IO3AT79\n9FMef/xxjj/++GJLcRynlVLIDqx91jL65CrEuyXNBeYDWwHHJs6dBvwozp89CVxmG+a9bgamA8/H\n5fYzgE+p2+uV1+qB2OMPB74h6XXgNWAVYdFILn4G9Mqn7bZKhw4dWLp0KV26dCm2FMdxWikFmwMz\ns3Z1lFc0cN1LhKjxuc4ZcGXcGrr/XODziePJwOSsOn0S+28DR9fRVhVh6DNz/DC1l+87juM4LUxb\nCCXlNJI+ffrQqVMn2rVrx6abbsoLL7xQbEmO4zg1tOkOTNIPCSGl1hGC+Z4BXAFsD6yM1d4wsxGS\nrgf+a2Y/SVy7g5l9t+WVtxzTp0+nW7duxZbhOI6zEW22A5N0EHAUsK+ZrZbUDdg8nj7FzLI/N34E\nzJZ0N2Fp/enAF1pMsOM4jlOLNtuBEb6ylpjZagAzWwIQokptjJl9Er+6boxFl5jZR/XdoFSNzPkm\ns5TE0KFDkcQZZ5zBt7/97QIrcxzHyR+Z1ecNbr1I6khYnt8B+BshTuIMSVXUHkJ83MwuSFz3DLDO\nzL5SR7tJI/N+l1x7S+EeopHs3KUdHTt2bLDekiVL6NatGx9++CFjx47lnHPOYZ99CuseWL58eV7a\nWhrXlQ7XlQ7XVZtBgwbNNLP9G6yYj1mstW4EE3UFcBnwPsHsXEXCaJ1VvxewgGC87thQ+63JyHzp\npZfaVVdd1fxisnBDZzpcVzpcVzrcyFzCmNk6M6sys0uBs4ETGrjkOmA8IWTVpQWWV1RWrFjBsmXL\navb/+te/0r9//yKrchzH2UCbnQOL0TvWm9nrsWgg8G8g529pSV8HtgPuIAw7zpF0uwXfWqtj0aJF\nNTm81q5dyze/+U0OP/zwIqtyHMfZQJvtwAhxF2+IyTTXAm8Q5q4eIEQKycyBLSGsVrwWGBE/b1fE\nNDA3ElKxtDp22WUX5syZU2wZjuM4ddJmOzAzmwkcnONURR2X7JF1/R8ISTDLmnXr1rH//vvTs2dP\nHnnkkWLLcRzHyZsWnwOTZJLuTBxvKum/kh7JqjctrvhLlo2X9E6MrfiSpMrEucmSFsRzczIBgeO5\nKkn7J46/EHUMy2q/h6TfSXpL0kxJz0gaHs9VSPo4K77j15rvzRSH6667jn79+hVbhuM4TmqKsYhj\nBdBfUvt4PISQBqWGOKy3L9BV0s5Z1//SQp6xY4HfxIj2GS6I584Dfl2PhkrCEvpkByhgKiGtyi5m\nth8hmn0yaO9TZjYwsf0tz2cuSRYuXMijjz7K6aefXmwpjuM4qSnWEOKfgSMJ802VwD1AMknkCcAf\ngUWETmSjlL5m9rqkTwmpTRZnnX4G6JnrxrGjGkHoOJ+StKWZrSLMZa0xs5qOz8z+DdzQmAeE4hqZ\n88mafN5553HllVfWrDZ0HMcpJ4rVgd0LXBKHDQcAt1G7A6skeLMWETq5jTowSfsCr5tZducFIYfY\n1Dru/WVggZm9GU3LRxDmsvYiJMWsj0NimpcMJ9iGVC8ZXSWRkbm+LKrLly9nwoQJfPbZZyxbtozZ\ns2ezdOnSksi8WqoZYF1XOlxXOlxXI8nHLNacGzFTM/AC8D/Az6md8bgHIVNyJkrILKB/3B9PGG58\nFfgMGJxodzIbTMbLM9fEc1VEczJwE/C/cf8Y4P64fw5heJJEvTnA8/G4RmO+WykbmceNG2c9e/a0\n3r17W48ePax9+/Z2yimnFFuaGzpT4rrS4brS4UbmunkY+AVh+DDJSYRhwQWSqoE+1M6s/Esz2yPW\nu0PSlolzFwC7EQLvTsm+Ycy0fALh66+aMDz4dUmdgBcJ824AWIgyPxjo3ugnLGEmTJjAwoULqa6u\n5t577+Wwww7jrrvuKrYsx3GcvClmB3YbcLmZzcsqrwQON7M+FhJOZhZT1MLCMvYXgJFZ5esJETM2\nyV5lCHwNmGNmO8b2exMyNx9HyP68paSzEvU7NPrpHMdxnIJStA7MzBaa2XXJMkl9gJ2AZxP1FgCf\nSDogRzOXA9+XVOs54ifoT4ELs+pXAg9llT0IfDNecxzw1bgc/znCV9xFibqHZC2jH5Hf05Y2FRUV\n7gFzHKfsaPFFHGa2UWhjM6sizFNBjtWDZpYZ2vtnVvlMNhiMR2Wde5DQOWFmFbnqxHMPE4YzMbP3\nyPG1l9DYJdc5x3Ecp+Vp08F8HcdxnPLFOzDHcRynLPEOzHEcxylL2mxG5pZA0jKCZ63U6EaIsl+K\nlKo215UO15UO11Wb3mbWoIWpzUajbyFetXzSYrcwkl4oRV1QutpcVzpcVzpcV+PwIUTHcRynLPEO\nzHEcxylLvAMrLL8ttoA6KFVdULraXFc6XFc6XFcj8EUcjuM4TlniX2CO4zhOWeIdmOM4jlOWeAdW\nICQdLulVSW9IGldsPRkkVUuaF4MRv1BEHbdJWixpfqJsG0mPS3o9/ty6RHSNl/ROIojzEUXQtaOk\n6ZJelvSipHNjeVHfWT26ivrOJG0p6TlJc6Kuy2L5zpL+Gd/XfZI2LxFdk2MQ8cz7GtiSuhL62kn6\nV0w2XPT31RDegRWAmHfsJuDrwOeBSkmfL66qWgwys4FF9ndMJmTOTjIOeMLM+gJPxOOWZjIb64KQ\nh25g3P7UwpoA1gL/Z2b9gAOB78b/p4r9zurSBcV9Z6uBw8xsH2AgcLikA4Eroq6+wIfAmBLRBXBB\n4n3NrruJgnIu8HLiuNjvq168AysMXwLeMLO3zGwNcC9wbJE1lRRm9v+AD7KKj2VDItIphPQ2LUod\nuoqOmb1nZrPi/jLCL5meFPmd1aOrqMTEvsvj4WZxM+Aw4IFYXoz3VZeuoiOpF3AkcGs8FkV+Xw3h\nHVhh6Am8nTheSAn8o44Y8FdJMyV9u9hisugRU9pkUttsV2Q9Sc6WNDcOMbb40GaSmDfvC4T0QiXz\nztsvSFQAAAYISURBVLJ0QZHfWRwOmw0sBh4H3gQ+MrO1sUpR/l1m6zKzzPv6WXxfv5S0RUvrAq4l\n5FBcH4+3pQTeV314B1YYlKOsJP7KAr4c86t9nTDcc2ixBZUBNwO7EoZ83gOuLpYQSR0Jee7OM7NP\niqUjmxy6iv7OzGydmQ0EehFGRfrlqtayqjbWJak/cDGwJ/BFYBtqJ9ItOJKOAhbHHIs1xTmqlsrv\nMcA7sEKxENgxcdwLeLdIWmphZu/Gn4sJ2am/VFxFtVgkaXuA+HNxkfUAYGaL4i+d9cAtFOmdSdqM\n0EncbWZ/iMVFf2e5dJXKO4taPiIkzD0Q6CopEwO2qP8uE7oOj0OxZmargdtp+ff1ZeAYSdWEKY/D\nCF9kJfO+cuEdWGF4HugbV/BsTsjy/HCRNSFpK0mdMvvAUGB+/Ve1KA8DI+P+SGBaEbXUkOkgIsMp\nwjuL8xGTgJfN7JrEqaK+s7p0FfudSeouqWvcbw98jTA/Nx0YEasV433l0vVK4o8QEeaZWvR9mdnF\nZtbLzPoQfl89aWanUOT31RAeiaNAxGXD1wLtgNvM7GdFloSkXQhfXRAyEfyuWLok3QNUENI1LAIu\nBaYCvwd2Av4DfMPMWnRBRR26KghDYQZUA2dk5p1aUNdXgKeAeWyYo/gBYb6paO+sHl2VFPGdSRpA\nWHTQjvCH+u/N7PL4b+BewjDdv4Bvxa+eYut6EuhOGLabDZyZWOzRokiqAMaa2VHFfl8N4R2Y4ziO\nU5b4EKLjOI5TlngH5jiO45Ql3oE5juM4ZYl3YI7jOE5Z4h2Y4ziOU5Z4B+Y4jUTSukT08NkxlFLa\nNrpK+k7zq6tp/xi1cDYESceVWPBqp5Xiy+gdp5FIWm5mHZvYRh/gETPrn/K6dma2rin3LgQxasOt\nhGd6oKH6jtMU/AvMcZqRGKj1KknPx8CsZ8TyjpKekDRLIR9bJjvBRGDX+AV3laSKTC6meN2NkkbF\n/WpJl0j6O/ANSbtKeiwGZn5K0p459IySdGPcnyzpZoX8XW9J+moMtPuypMmJa5ZLujpqfUJS91g+\nUNKz8bkeygTolVQl6eeSZhBi+B0DXBWfaVdJ/xvfxxxJD0rqkNBzvaSno54RCQ0Xxvc0R9LEWNbg\n8zptDDPzzTffGrEB6whRE2YDD8WybwM/ivtbAC8AOxMin3SO5d2ANwhRF/oA8xNtVhC+XjLHNwKj\n4n41cGHi3BNA37h/ACH8T7bGUcCNcX8yIaqCCGlYPgH2JvwhOxMYGOsZcErcvyRx/Vzgq3H/cuDa\nuF8F/Cpxz8nAiMTxton9nwLfS9S7P97/84QURBACTT8NdIjH2+T7vL61rS0TpNFxnPSstBBVPMlQ\nYEDia6IL0JcQ4PnnCtH/1xPSUvRoxD3vg5ro7wcD94fweUDoMBvij2ZmkuYBi8xsXmzvRUJnOjvq\nuy/Wvwv4g6QuQFczmxHLpxA6n1q66qC/pJ8CXYGOwF8S56ZaCPj7kqTM+/gacLuZfQpgZh804Xmd\nVox3YI7TvIjwhfGXWoVhGLA7sJ+ZfaYQ9XvLHNevpfbQfnadFfHnJoRcTWlTz2fi2K1P7GeO6/p9\nkM9E+Yp6zk0GjjOzOfE9VOTQAxvSdyjHPRv7vE4rxufAHKd5+QtwlkKKESTtHiP/dyHkW/pM0iCg\nd6y/DOiUuP7fwOclbRG/egbnuomFnFsLJH0j3keS9mmmZ9iEDRHIvwn83cw+Bj6UdEgsPxWYketi\nNn6mTsB78Z2cksf9/wqMTsyVbVPg53XKFO/AHKd5uRV4CZglaT7wG/5/e3eIm1AQxGH8+58FWctt\nKhBcgUoMtucg6RGaVONoSEmb1Fcj0VMxz1QUBSSbfD/5XiabNW8y2X0zXdlsgXmSPf0R/waoqhOw\nS/KZ5Lmqfuju8scp5nBhrUdgmeQD+KLPta7hDDwkeafnQm2m5wv6csaR7jS/+Sf+BXhKckgyA9Z0\n1/w3pn1fUlWv9JiYfXpy8Wp6dav9alBeo5f0xzV+D5DuwQpMkjQkKzBJ0pCswCRJQzKBSZKGZAKT\nJA3JBCZJGpIJTJI0pF8oHHcOkw4x1wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110400080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = lgb.plot_importance(gbm, max_num_features=25)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
